Gitlab full - fixes

Pipfile.lock

@@ -52,11 +52,11 @@
         },
         "authlib": {
             "hashes": [
-                "sha256:b61c6c6fd230c4ba8602fd85ee9a40e6dc859387699a1cd1f7247c4b109dcc17",
-                "sha256:eda3e5af921a368091fef721d6d169bcff2aa0003d05113bc26e127f58c9a5e8"
+                "sha256:3a226f231e962a16dd5f6fcf0c113235805ba206e294717a64fa8e04ae3ad9c4",
+                "sha256:9741db6de2950a0a5cefbdb72ec7ab12f7e9fd530ff47219f1530e79183cbaaf"
             ],
             "index": "pypi",
-            "version": "==0.10"
+            "version": "==0.11"
         },
         "billiard": {
             "hashes": [

management/data/data_test.py

@@ -4,7 +4,7 @@ from flask import Flask
 from flask_sqlalchemy import SQLAlchemy
 
 from management.data.shared import DataFactory
-from portal.database.models import Review, Secret, Submission, SubmissionState
+from portal.database.models import Review, Secret, SubmissionState
 from portal.tools import time
 
 
@@ -23,8 +23,10 @@ def init_test_data(app: Flask, db: SQLAlchemy):
         lecturer1 = factory.create_user(username='lecturer1', name='Courses Owner', uco=1010)
 
         # courses
-        test_course1 = factory.create_course(codename='testcourse1', name='Test Course One')
-        test_course2 = factory.create_course(codename='testcourse2', name='test Course Two')
+        test_course1 = factory.create_course(codename='testcourse1', name='Test Course One',
+                                             faculty_id=100)
+        test_course2 = factory.create_course(codename='testcourse2', name='test Course Two',
+                                             faculty_id=100)
 
         # groups
         tc1_students = factory.create_group(course=test_course1, name="seminar01")

management/data/shared.py

@@ -129,12 +129,15 @@ class DataFactory:
     def create_worker(self, name: str, url: str) -> Worker:
         return self.__create_entity(Worker, name=name, url=url)
 
-    def create_course(self, codename, name=None, token=None) -> Course:
+    def create_course(self, codename, name=None, token=None, faculty_id=None) \
+            -> Course:
         name = name or codename
         desc = name + "'s description"
         course = self.__create_entity(
             Course, codename=codename, name=name, description=desc)
         course.notes_access_token = token or f"{codename}_token"
+        if faculty_id is not None:
+            course.faculty_id = faculty_id
         return course
 
     def create_group(self, course: Course, name: str) -> Group:

portal/__init__.py

 """
 Main Portal module
 """
-
+import logging
 import os
 from typing import Union
 
@@ -28,7 +28,7 @@ storage_wrapper = Storage()
 migrate = Migrate(db=db)
 ldap_wrapper = LDAPWrapper()
 is_api_factory = IsApiFactory()
-log = logger.get_logger(__name__)
+log = logging.getLogger(__name__)
 
 
 def configure_app(app: Flask, env: str = None,
@@ -116,7 +116,7 @@ def create_app(environment: str = None):
 
     Returns(Flask): Flask application instance
     """
-    app = Flask('portal')
+    app = Flask(__name__)
     # app configuration
     configure_app(app, env=environment)
     _log_config(app)
@@ -129,7 +129,6 @@ def create_app(environment: str = None):
         rest.gitlab.register_gitlab_app(app)
 
     rest.register_namespaces(app)
-
     return app
 
 

portal/config.py

@@ -22,10 +22,16 @@ log = logger.get_logger(__name__)
 class Config(object):
     """Base configuration
     """
+    ENV = os.getenv('ENV', 'development')
+    FLASK_ENV = os.getenv('FLASK_ENV', ENV)
+
     # Base dirs
     PROJECT_ROOT = paths.ROOT_DIR
     APP_DIR = paths.ROOT_DIR / 'portal'
 
+    # FRONTEND
+    FRONTEND_URL = os.getenv('FRONTEND_URL', 'http://localhost:4200')
+
     # Secrets and JWT Config
     SECRET_KEY = os.environ.get('SECRET_KEY') or 'you-will-never-guess'
     JWT_SECRET_KEY = os.environ.get('JWT_SECRET_KEY') or 'super-safe'
@@ -82,6 +88,9 @@ class Config(object):
     UPLOAD_FOLDER = os.getenv('PORTAL_UPLOAD_FOLDER', f"{PORTAL_STORAGE_BASE_DIR}/upload")
     GITPYTHON_CUSTOM_SSH_KEY = os.getenv('GITPYTHON_CUSTOM_SSH_KEY', None)
 
+    # IS MUNI NOTES Integration
+    IS_API_DOMAIN = os.getenv('IS_API_DOMAIN', None)
+
 
 class DevelopmentConfig(Config):
     """Development configuration
@@ -101,6 +110,7 @@ class ProductionConfig(Config):
     LOG_LEVEL_GLOBAL = os.getenv('LOG_LEVEL_GLOBAL', 'INFO')
     LOG_LEVEL_FILE = os.getenv('LOG_LEVEL_FILE', LOG_LEVEL_GLOBAL)
     LOG_LEVEL_CONSOLE = os.getenv('LOG_LEVEL_CONSOLE', LOG_LEVEL_GLOBAL)
+    FLASK_ENV = os.getenv('FLASK_ENV', 'production')
 
 
 class TestConfig(Config):
@@ -109,6 +119,7 @@ class TestConfig(Config):
     SQLALCHEMY_TRACK_MODIFICATIONS = False
     TESTING = True
     ENV = 'development'
+    FLASK_ENV = ENV
     DEBUG = False
     SQLALCHEMY_DATABASE_URI = 'sqlite://'
     # SQLALCHEMY_ECHO = True
@@ -133,7 +144,7 @@ class TestConfig(Config):
     GITLAB_KONTR_USERNAME = os.getenv('GITLAB_KONTR_USERNAME', 'admin')
     GIT_REPO_BASE = os.getenv('GIT_REPO_BASE', f"git@{GITLAB_BASE_DOMAIN}")
     GITLAB_URL = f'https://{GITLAB_BASE_DOMAIN}'
-    FRONTEND_URL = os.getenv('FRONTEND_URL', 'http://localhost:4200')
+    IS_API_DOMAIN = 'is.muni.local'
 
 
 # pylint: enable=too-few-public-methods

portal/database/models.py

@@ -731,11 +731,9 @@ class Role(db.Model, EntityBase, NamedMixin):
     evaluate_submissions = db.Column(db.Boolean, default=False, nullable=False)
 
     read_submissions_all = db.Column(db.Boolean, default=False, nullable=False)
-    read_submissions_groups = db.Column(
-        db.Boolean, default=False, nullable=False)
+    read_submissions_groups = db.Column(db.Boolean, default=False, nullable=False)
     read_submissions_own = db.Column(db.Boolean, default=False, nullable=False)
-    read_all_submission_files = db.Column(
-        db.Boolean, default=False, nullable=False)
+    read_all_submission_files = db.Column(db.Boolean, default=False, nullable=False)
 
     read_reviews_all = db.Column(db.Boolean, default=False, nullable=False)
     read_reviews_groups = db.Column(db.Boolean, default=False, nullable=False)

portal/facade/gitlab_facade.py

 import logging
 from typing import List
 
+import flask
 from flask import make_response
 from gitlab.v4 import objects
 from werkzeug.utils import redirect
@@ -92,11 +93,19 @@ class GitlabFacade(GeneralFacade):
         return new_user
 
     def _make_response(self, token, token_type='oauth', user_info=None, redir=True):
-        frontend = self._config.get('FRONTEND_URL')
+        params = {
+            'gitlab_token': token,
+            'gitlab_token_type': token_type,
+            'gitlab_username': user_info.username,
+            'username': user_info.username,
+        }
+
+        return flask.jsonify(params) if not redir else self._make_login_response(params)
+
+    def _make_login_response(self, params):
+        frontend = self._config.get('FRONTEND_URL') or 'http://localhost:4200'
         front_gl_login = frontend + '/gitlabLogin'
-        login_response = make_response(redirect(front_gl_login)) if redir else make_response()
-        login_response.set_cookie('gitlab_token', token)
-        login_response.set_cookie('gitlab_token_type', token_type)
-        login_response.set_cookie('gitlab_username', user_info.username)
-        login_response.set_cookie('username', user_info.username)
+        login_response = make_response(redirect(front_gl_login))
+        for (key, val) in params.items():
+            login_response.set_cookie(key, val)
         return login_response

portal/facade/submissions_facade.py

@@ -129,24 +129,25 @@ class SubmissionsFacade(GeneralCRUDFacade):
             user=submission.user)
         return note
 
-    def is_write_notepad(self, submission: Submission, content: str = None):
+    def is_write_notepad(self, submission: Submission, data: dict = None):
         """Write IS MUNI notepad content for the submission
         Args:
+            data: Notepad data
             submission(Submission): Submission instance
-            content(str): Notepad content that will be written
         """
         log.info(f"[IS] Writing notepad content for {submission.log_name} "
-                 f"by {self.client_name}: {content}")
-        self._submission_email(content, submission)
-        return self._services.is_api(submission.course).write_note(
-            project=submission.project, user=submission.user, content=content
-        )
+                 f"by {self.client_name}: {data}")
+        content = data.get('content') if isinstance(data, dict) else data
+        response = self._services.is_api(submission.course) \
+            .write_note(project=submission.project, user=submission.user, content=content)
+        self._submission_email(context=dict(content=content), submission=submission)
+        return response
 
-    def _submission_email(self, content, submission):
+    def _submission_email(self, context, submission):
         self._services.emails.notify_user(
             user=submission.user,
             template='is_notepad_updated',
-            content=content,
+            context=context,
             submission=submission,
             submission_url=self.urls.submission(submission=submission)
         )

portal/logger.py

@@ -6,9 +6,10 @@ import logging
 import os
 from logging.config import dictConfig
 
+from dotenv import load_dotenv
+
 from portal.tools import paths
 
-from dotenv import load_dotenv
 load_dotenv()
 
 
@@ -44,6 +45,8 @@ class Logging:
             'login_file': self.get_logger_file('login'),
             'submit_file': self.get_logger_file('submit'),
             'files_access_file': self.get_logger_file('files_access'),
+            'gitlab_file': self.get_logger_file('gitlab'),
+            'is_notes_files': self.get_logger_file('is_notes'),
         }
 
     @property
@@ -101,13 +104,13 @@ class Logging:
                 'handlers': ['console'], 'level': self.global_log_level, 'propagate': True
             },
             'flask': {
-                'handlers': ['flask_console', 'flask_file'],
+                'handlers': ['flask_file'],
                 'level': self.global_log_level,
                 'propagate': True
             },
-            'werkzeug': {
-                'handlers': ['flask_console', 'flask_file'],
-                'level': self.global_log_level, 'propagate': True,
+            'werkzeug   ': {
+                'handlers': ['flask_file'],
+                'level': 'ERROR', 'propagate': True,
                 'disabled': True,
             },
             'git': {
@@ -116,10 +119,13 @@ class Logging:
                 'propagate': True
             },
             'gitlab': {
-                'handlers': ['console'],
+                'handlers': ['console', 'portal_file', 'gitlab_file'],
                 'level': self.global_log_level,
                 'propagate': True
             },
+            'pytest': {
+                'handlers': ['console'], 'level': 'ERROR', 'propagate': True
+            },
         }
 
     @property

portal/rest/error_handlers.py

@@ -20,7 +20,7 @@ def load_errors(app: Flask):
 
 def send_response(body):
     response: flask.Response = flask.jsonify(body)
-    response.headers['Access-Control-Allow-Origin'] = '*'
+    # response.headers['Access-Control-Allow-Origin'] = '*'
     return response
 
 

portal/rest/gitlab.py

@@ -74,6 +74,7 @@ class GitlabProjects(CustomResource):
         self.facades.gitlab.check_gl_enabled()
         args = {**self.request.args}
         args['membership'] = args.get('membership') or True
+        args['all'] = args.get('all') or True
         projects: List[objects.Project] = self.facades.gitlab.list_projects(**args)
         return jsonify([proj.attributes for proj in projects])
 
@@ -101,7 +102,8 @@ class GitlabProjectMembers(CustomResource):
     @jwt_required
     def post(self, pname: str):
         self.facades.gitlab.check_gl_enabled()
-        return self.facades.gitlab.add_member_to_project(project_name=pname)
+        result = self.facades.gitlab.add_member_to_project(project_name=pname)
+        return result
 
 
 @gitlab_namespace.route('/service_token')

portal/rest/rest_helpers.py

 """
 Helpers to work and process the services requests and responses
 """
-from typing import Optional, List
+import logging
+from typing import Dict, List, Optional, Union
 
 import flask
 from werkzeug.datastructures import ImmutableMultiDict
 
-from portal.database import Course, Project, User, Group, Role
+from portal.database import Course, Group, Project, Role, User
 from portal.rest.schemas import SCHEMAS
 from portal.service.errors import DataMissingError
 
+log = logging.getLogger(__name__)
+
 
 def parse_request_data(schema=None, action: str = None, resource: str = None,
                        partial=False) -> dict:
@@ -31,7 +34,7 @@ def parse_request_data(schema=None, action: str = None, resource: str = None,
     return schema.load(json_data, partial=partial)[0]
 
 
-def require_data(action: str, resource: str) -> dict:
+def require_data(action: str, resource: str) -> Union[Dict, str, List]:
     """Parses data from json and if they are missing, throws an exception
     Args:
         action(str): Name of the action
@@ -42,6 +45,7 @@ def require_data(action: str, resource: str) -> dict:
     """
     json_data = flask.request.get_json()
     if not json_data:
+        log.warning(f"[DATA] Data are missing in the request: {flask.request}")
         raise DataMissingError(action=action, resource=resource)
     return json_data
 

portal/rest/submissions.py

@@ -173,10 +173,9 @@ class SubmissionResultFilesTree(CustomResource):
     @jwt_required
     def get(self, sid: str):
         submission = self.find.submission(sid)
-        course = submission.project.course
         log.debug(f"[REST] Get submission result tree"
                   f" by {self.client.log_name}: {submission.log_name}")
-        self.permissions(course=course).require.read_submission_group(submission)
+        self.permissions(submission=submission).require.read_submission_group(submission)
         log_files_access(f"Result files tree", submission=submission, client=self.client)
         return self.facades.submissions.result_files_tree(submission)
 
@@ -188,10 +187,9 @@ class SubmissionResultFiles(CustomResource):
     @jwt_required
     def get(self, sid: str):
         submission = self.find.submission(sid)
-        course = submission.project.course
         log.debug(f"[REST] Get submission result files"
                   f" by {self.client.log_name}: {submission.log_name}")
-        self.permissions(course=course).require.read_submission_group(submission)
+        self.permissions(submission=submission).require.read_submission_group(submission)
         log_files_access(f"Result files", submission=submission, client=self.client)
         return self.facades.submissions.get_result_files(submission)
 
@@ -200,8 +198,7 @@ class SubmissionResultFiles(CustomResource):
     def post(self, sid: str):
         submission = self.find.submission(sid)
         # authorization
-        course = submission.project.course
-        self.permissions(course=course).require.permissions(['evaluate_submissions'])
+        self.permissions(submission=submission).require.permissions(['evaluate_submissions'])
         # todo: authorize worker
         log.info(f"[REST] Upload submission results for {submission.log_name} by "
                  f"{self.client.log_name}")
@@ -219,8 +216,7 @@ class SubmissionResubmit(CustomResource):
     def post(self, sid: str):
         source_submission = self.find.submission(sid)
         # authorization
-        course = source_submission.project.course
-        perm_service = self.permissions(course=course)
+        perm_service = self.permissions(submission=source_submission)
         perm_service.require.permissions(permissions=['resubmit_submissions'])
 
         data = rest_helpers.parse_request_data(action='resubmit', resource='submission')
@@ -260,8 +256,7 @@ class SubmissionWorkerParams(CustomResource):
     def get(self, sid: str):
         submission = self.find.submission(sid)
         # authorization
-        course = submission.project.course
-        perm_service = self.permissions(course=course)
+        perm_service = self.permissions(submission=submission)
         perm_service.require.read_submission(submission)
         log.debug(f"[REST] Worker params for submission {submission.log_name}"
                   f": {submission.worker_parameters}")
@@ -279,8 +274,7 @@ class SubmissionCancel(CustomResource):
     def post(self, sid: str):
         submission = self.find.submission(sid)
         # authorization
-        course = submission.project.course
-        perm_service = self.permissions(course=course)
+        perm_service = self.permissions(submission=submission)
         perm_service.require.sysadmin_or_self(submission.user.id)
         # create new submission by copying files from the source submission in
         # storage
@@ -297,10 +291,9 @@ class SubmissionReview(CustomResource):
     # @submissions_namespace.response(200, 'Submissions review', model=review_schema)
     def get(self, sid: str):
         submission = self.find.submission(sid)
-        course = submission.project.course
         log.debug(f"[REST] Get submission review"
                   f" by {self.client.log_name}: {submission.log_name}")
-        self.permissions(course=course).require.read_submission(submission)
+        self.permissions(submission=submission).require.read_submission(submission)
         return SCHEMAS.dump('review', submission.review)
 
     @jwt_required
@@ -308,8 +301,7 @@ class SubmissionReview(CustomResource):
     # @submissions_namespace.response(201, 'Submissions review created', model=review_schema)
     def post(self, sid: str):
         submission = self.find.submission(sid)
-        course = submission.project.course
-        perm_service = self.permissions(course=course)
+        perm_service = self.permissions(submission=submission)
         perm_service.require.write_review_for_submission(submission)
         data = rest_helpers.parse_request_data(action='create', resource='review')
         log.info(f"[REST] Create submission review {submission.log_name} by "
@@ -318,6 +310,31 @@ class SubmissionReview(CustomResource):
         return SCHEMAS.dump('review', review), 201
 
 
+@submissions_namespace.route('/<string:sid>/review/is_muni/notepad')
+@submissions_namespace.param('sid', 'Submission id')
+@submissions_namespace.response(404, 'Submissions not found')
+class SubmissionISNotes(CustomResource):
+    @jwt_required
+    # @submissions_namespace.response(200, 'Submissions review', model=review_schema)
+    def get(self, sid: str):
+        submission = self.find.submission(sid)
+        self.permissions(submission=submission).require.evaluate_submissions()
+        log.debug(f"[REST] Get IS MUNI Notepad for project {submission.project.name}"
+                  f" by {self.client.log_name}: {submission.log_name}")
+        return self.facades.submissions.is_read_notepad(submission=submission)
+
+    @jwt_required
+    @access_log
+    # @submissions_namespace.response(201, 'Submissions review created', model=review_schema)
+    def post(self, sid: str):
+        submission = self.find.submission(sid)
+        self.permissions(submission=submission).require.evaluate_submissions()
+        data = rest_helpers.require_data(action='update', resource='is-muni-notepad')
+        log.info(f"[REST] Update notepads content {submission.log_name} by "
+                 f"{self.client.log_name}: {data}")
+        return self.facades.submissions.is_write_notepad(submission=submission, data=data)
+
+
 @submissions_namespace.route('/<string:sid>/review/<string:rid>')
 @submissions_namespace.param('sid', 'Submission id')
 @submissions_namespace.param('rid', 'Review item id')

portal/service/emails.py

@@ -30,6 +30,7 @@ class EmailMessage:
         self._subject = JinjaTemplate(subject)
         self._body = JinjaTemplate(body)
         self._params = kwargs
+        self._message = None
 
     @property
     def subject(self) -> JinjaTemplate:
@@ -57,6 +58,15 @@ class EmailMessage:
         )
         return message
 
+    @property
+    def message(self) -> Message:
+        if self._message is None:
+            self._message = self.create_message()
+        return self._message
+
+    def __str__(self):
+        return str(dict(subject=self.subject.template_text))
+
 
 def _read_yml(full_path):
     with open(str(full_path), 'r') as stream:

portal/service/gitlab.py

@@ -118,8 +118,14 @@ class GitlabService(GeneralService):
         members = self._project_members(project_name=project_name)
         member_id = member_id or self.kontr_gl_user.id
         access_level = access_level or gitlab.REPORTER_ACCESS
-        params = {'user_id': member_id.id, 'access_level': access_level}
-        return members.create(params, **kwargs)
+        params = {'user_id': member_id, 'access_level': access_level}
+        log.info(f"[GL_MEMBER] Adding member to a project {project_name} - {params}")
+        try:
+            return members.create(params, **kwargs)
+        except gitlab.exceptions.GitlabCreateError as ex:
+            log.warning(f"[GL_MEMBER] User is already a member: {ex}")
+            raise errors.PortalAPIError(
+                400, f"User is already a member of a project {project_name}")
 
     @property
     def kontr_gl_user(self) -> objects.CurrentUser:

portal/service/is_api_service.py

@@ -37,7 +37,7 @@ class IsApiService(GeneralService):
 
     @property
     def is_api(self) -> IsApiWrapper:
-        return self.is_factory.get_instance(self.course)
+        return self.is_factory.get_instance(course=self.course)
 
     def import_students(self, role_name: str = 'student'):
         """Import students to course from IS MU

portal/service/permissions.py

@@ -107,6 +107,9 @@ class PermissionServiceCheck:
     def create_submission_other(self):
         return self.permissions(['create_submissions_other'])
 
+    def evaluate_submissions(self):
+        return self.permissions(['evaluate_submissions'])
+
 
 class PermissionServiceRequire:
     def __init__(self, service: 'PermissionsService'):
@@ -163,6 +166,9 @@ class PermissionServiceRequire:
         ]
         self.any_check(*checks)
 
+    def evaluate_submissions(self):
+        self.permissions(['evaluate_submissions'])
+
     def read_submission(self, submission=None):
         submission = submission or self.service.submission
         checks = [

portal/tools/is_api_adapter.py

@@ -94,11 +94,11 @@ class IsApiFactory:
 
     @property
     def is_domain(self) -> str:
-        return self._app.get('IS_API_DOMAIN')
+        return self._app.config.get('IS_API_DOMAIN')
 
     @property
     def faculty_id(self) -> str:
-        return self._app.get('IS_API_FACULTY_ID')
+        return self._app.config.get('IS_API_FACULTY_ID')
 
     def get_instance(self, course: 'Course') -> IsApiWrapper:
         params = self.__extract_params(course)
@@ -112,7 +112,7 @@ class IsApiFactory:
             raise EnvironmentError("IS API is not enabled.")
         if not course.notes_access_token:
             raise errors.AccessTokenForCourseMissingError(course)
-        params = dict(domain=self.is_domain, faculty_id=self.faculty_id, )
+        params = dict(domain=self.is_domain, faculty_id=self.faculty_id, course=course)
         return params
 
     def is_api_enabled(self) -> bool:

pytest.ini

+[pytest]
+log_format = %(asctime)s %(levelname)s %(message)s
+log_date_format = %Y-%m-%d %H:%M:%S
+addopts = -rsxX -q
\ No newline at end of file

tests/rest/test_gitlab.py

@@ -214,7 +214,10 @@ def test_set_gitlab_service_cookies(client, gl_api_url, gl_user, gl_cookies):
 
     # 'https://gitlab.local/api/v4/user'
     response = rest_tools.make_request(client, f'/gitlab/service_token')
-    assert response.status_code == 200
-    cookies = response.headers.get_all('Set-Cookie')
-    assert cookies
-    assert len(cookies) == 4
+    assert_response(response)
+    res_proj = rest_tools.extract_data(response)
+    assert len(res_proj) == 4
+    assert res_proj['username'] == gl_cookies['username']
+    assert res_proj['gitlab_username'] == gl_cookies['gitlab_username']
+    assert res_proj['gitlab_token'] == gl_cookies['gitlab_token']
+    assert res_proj['gitlab_token_type'] == gl_cookies['gitlab_token_type']