Log name support, Submission course hybrid property

app.py

@@ -66,9 +66,10 @@ def cli_init_data(env):
 @click.argument('name')
 @click.option('-p', '--password', help='Users password', prompt=True, hide_input=True,
               confirmation_prompt=True)
-def cli_users_create(name, password):
+@click.option('-s', '--secret', help="User's secret")
+def cli_users_create(name, password, secret):
     log.info(f"[CMD] Create User: {name}")
-    user = manager.create_admin_user(name, password)
+    user = manager.create_admin_user(name, password, secret)
     log.info(f"[CMD] Created User: {user}")
 
 

docker_run.sh

@@ -3,6 +3,7 @@
 ENVIRONMENT="${ENVIRONMENT:-dev}"
 ADMIN_USER="${ADMIN_USER:-admin}"
 ADMIN_PASSWORD="${ADMIN_PASSWORD:-789789}"
+ADMIN_SECRET="${ADMIN_SECRET:-devel-admin-secret}"
 
 # INIT
 echo "[INIT] Initializing the database"
@@ -10,7 +11,7 @@ flask db upgrade
 echo "[INIT] Initializing data for env: $ENVIRONMENT"
 flask data init ${ENVIRONMENT}
 echo "[INIT] Initializing admin user: ${ADMIN_USER}"
-flask users create ${ADMIN_USER} --password ${ADMIN_PASSWORD}
+flask users create ${ADMIN_USER} --password ${ADMIN_PASSWORD} --secret ${ADMIN_SECRET}
 
 # RUN THE SERVER
 gunicorn -b 0.0.0.0:8000 --access-logfile - --reload "app:app"

management/data/__init__.py

 import datetime
+
 from flask import Flask
 from flask_sqlalchemy import SQLAlchemy
 
 from management.data import shared
-from management.data.data_test import init_test_data
 from management.data.data_dev import init_dev_data
 from management.data.data_prod import init_prod_data
-from portal.database.models import Course, Role, User, Submission, SubmissionState
+from management.data.data_test import init_test_data
+from portal.database.models import Course, Role, Secret, Submission, SubmissionState, User
 from portal.service import general
 from portal.service.general import write_entity
 from portal.tools import time
@@ -65,11 +66,12 @@ class DataManagement(object):
             self.db.session.commit()
         return user
 
-    def create_admin_user(self, name: str, password: str) -> User:
+    def create_admin_user(self, name: str, password: str, secret: str = None) -> User:
         """Creates new admin user with username
         Args:
             name(str): Username of the user
             password(str): User's password
+            secret(str): User's secret
         Returns(User): Created user
         """
         with self.app.app_context():
@@ -77,8 +79,9 @@ class DataManagement(object):
                 username=name,
                 name=name,
                 admin=True,
-                password=password
-                )
+                password=password)
+            if secret:
+                user.secrets.append(Secret(name='admin-secret', value=secret))
             self.db.session.commit()
         return user
 
@@ -158,5 +161,3 @@ class DataManagement(object):
             for submission in Submission.query.all():
                 Submission.query.filter_by(id=submission.id).delete()
             self.db.session.commit()
-
-

management/data/shared.py

@@ -148,7 +148,7 @@ class DataFactory(object):
         codename = codename or name
         project = self.__create_entity(Project, course=course, name=name,
                                        codename=codename, description=desc)
-        log.debug(f"[CREATE] Project config: ({project.id}): {config}")
+        log.debug(f"[CREATE] Project config: {project.log_name}: {config}")
         project.set_config(**config)
         return project
 

portal/async_celery/submission_processor.py

-import json
 import random
 from pathlib import Path
-from typing import Union
 
 from storage import UploadedEntity
 
@@ -48,11 +46,11 @@ class SubmissionProcessor:
         general.write_entity(self.submission)
 
     def submission_enqueue_ended(self):
-        log.info(f"[ASYNC] Submission enqueue ended: {self.submission}")
+        log.info(f"[ASYNC] Submission enqueue ended {self.submission.log_name}: {self.submission}")
         self.reset_task_id(state=SubmissionState.QUEUED)
 
     def get_delay_for_submission(self):
-        log.debug(f"[ASYNC] Submission delay: {self.submission}")
+        log.debug(f"[ASYNC] Submission delay {self.submission.log_name}: {self.submission}")
         time_wait = self.project.config.submissions_cancellation_period
         return time_wait
 
@@ -63,42 +61,43 @@ class SubmissionProcessor:
         start_processing_submission.apply_async(args=args, countdown=delay)
 
     def submission_store_ended(self, version: str):
-        log.info(f"[ASYNC] Submission preparation ended: {self.submission}")
+        log.info(f"[ASYNC] Submission preparation ended {self.submission.log_name}: "
+                 f"{self.submission}")
         self.submission.source_hash = version
         self.reset_task_id(state=SubmissionState.READY)
 
     def download_submission(self):
         file_params = self.params['file_params']
-        log.info(f"[ASYNC] Uploading submission: {self.submission} with {file_params}")
+        log.info(f"[ASYNC] Uploading submission: {self.submission.log_name} with {file_params}")
         updated_entity: UploadedEntity = self.storage. \
             submissions.create(entity_id=self.submission.id, **file_params)
         self.submission_store_ended(version=updated_entity.version)
 
     def clone(self, target):
-        log.info(f"[ASYNC] Cloning submission: {self.submission} to {target}")
+        log.info(f"[ASYNC] Cloning submission: {self.submission.log_name} to {target.log_name}")
         self.storage.submissions.clone(self.submission.id, target.id)
         self.submission_store_ended(version=self.submission.source_hash)
 
     def send_to_worker(self):
         # TODO: implement processing
-        log.info(f"[ASYNC] Sending submission to worker: {self.submission}")
+        log.info(f"[ASYNC] Sending submission to worker: {self.submission.log_name}")
         worker = self.schedule_submission_to_worker()
         self.execute_submission(worker)
 
     def upload_result(self, path, file_params):
         log.info(f"[ASYNC] Uploading result for the submission "
-                 f"{self.submission.id} with {file_params}")
+                 f"{self.submission.log_name} with {file_params}")
         self.storage.results.create(entity_id=self.submission.id, **file_params)
         Path(path).unlink()
         self.reset_task_id(SubmissionState.FINISHED)
 
     def process_submission(self):
-        log.info(f"[ASYNC] Processing submission: {self.submission}")
+        log.info(f"[ASYNC] Processing submission {self.submission.log_name}")
         self.download_submission()
         self.dispatch_submission_processing()
 
     def revoke_task(self):
-        log.info(f'[ASYNC] Submission processing cancelled {self.submission}')
+        log.info(f'[ASYNC] Submission processing cancelled {self.submission.log_name}')
         task_id = self.submission.async_task_id
         if task_id:
             self.celery.control.revoke(task_id=task_id, terminate=True)
@@ -124,5 +123,5 @@ class SubmissionProcessor:
         worker_client.execute_submission(self.submission)
 
     def _worker_not_available(self):
-        log.warning(f"[PROC] Worker is no available for submission: {self.submission}")
+        log.warning(f"[PROC] Worker is no available for submission: {self.submission.log_name}")
         pass

portal/async_celery/tasks.py

@@ -14,7 +14,9 @@ def process_submission(new_submission_id: str):
     project = new_submission.project
     course = project.course
     if not project.config.test_files_commit_hash:
+        log.warning(f"Project test files not found: {project.log_name}")
         update_project_test_files(course_id=course.id, project_id=project.id)
+    new_submission = find_submission(new_submission_id)
     processor = submission_processor.SubmissionProcessor(new_submission)
     processor.process_submission()
 
@@ -45,9 +47,9 @@ def start_processing_submission(submission_id: str, submission_params):
 
 @celery_app.task(name='update-project-test-files')
 def update_project_test_files(course_id: str, project_id: str):
-    log.info(f"[ASYNC] Updating test files for project: {project_id}")
     course = find_course(course_id)
     project = find_project(course, project_id)
+    log.info(f"[ASYNC] Updating test files for project: {project.log_name}")
     params = {
         'from_dir': project.config.test_files_subdir,
         'source': {
@@ -57,5 +59,5 @@ def update_project_test_files(course_id: str, project_id: str):
     }
     updated_entity: UploadedEntity = storage.test_files.update(entity_id=project.id, **params)
     project.config.test_files_commit_hash = updated_entity.version
-    log.debug(f"Updated project config: {project.config}")
+    log.debug(f"Updated project config {project.log_name}: {project.config}")
     write_entity(project.config)

portal/database/models.py

@@ -71,6 +71,10 @@ class Client(db.Model):
         'polymorphic_on': type
     }
 
+    @property
+    def log_name(self):
+        return f"{self.id} ({self.codename})"
+
     def __init__(self, client_type: ClientType):
         self.type = client_type
 
@@ -157,6 +161,10 @@ class Secret(db.Model):
         self.value = generate_password_hash(value)
         self.expires_at = expires_at
 
+    @property
+    def log_name(self):
+        return f"{self.id} ({self.name})"
+
 
 class User(EntityBase, Client):
     """User entity model
@@ -326,6 +334,10 @@ class Course(db.Model, EntityBase, NamedMixin):
         db.UniqueConstraint('codename', name='course_unique_codename'),
     )
 
+    @property
+    def log_name(self):
+        return f"{self.id} ({self.codename})"
+
     def __init__(self, name: str = None, codename: str = None,
                  description: str = None) -> None:
         """Creates course instance
@@ -441,6 +453,10 @@ class Project(db.Model, EntityBase, NamedMixin):
         db.UniqueConstraint('course_id', 'codename', name='p_course_unique_name'),
     )
 
+    @property
+    def log_name(self):
+        return f"{self.id} ({self.course.codename}/{self.codename})"
+
     def state(self, timestamp=time.current_time()) -> ProjectState:
         """Gets project state based on the timestamp
         Args:
@@ -666,6 +682,10 @@ class Role(db.Model, EntityBase, NamedMixin):
                                   cascade="all, delete-orphan",
                                   passive_deletes=True, uselist=False)
 
+    @property
+    def log_name(self):
+        return f"{self.id} ({self.course.codename}/{self.codename})"
+
     def set_permissions(self, **kwargs):
         """Sets permissions for the role
         Args:
@@ -792,6 +812,10 @@ class Group(db.Model, EntityBase, NamedMixin):
         db.UniqueConstraint('course_id', 'codename', name='g_course_unique_name'),
     )
 
+    @property
+    def log_name(self):
+        return f"{self.id} ({self.course.codename}/{self.codename})"
+
     def __init__(self, course: Course, name: str = None,
                  description: str = None, codename: str = None):
         """Creates instance of the group
@@ -865,6 +889,15 @@ class Submission(db.Model, EntityBase):
 
     async_task_id = db.Column(db.String(length=36))
 
+    @property
+    def log_name(self):
+        return f"{self.id} ({self.course.codename}/{self.project.codename} for " \
+               f"{self.user.username})"
+
+    @hybrid_property
+    def course(self):
+        return self.project.course
+
     def change_state(self, new_state):
         # open to extension (state transition validation, ...)
         self.state = new_state
@@ -1001,6 +1034,10 @@ class Worker(EntityBase, Client):
         'polymorphic_identity': ClientType.WORKER,
     }
 
+    @property
+    def log_name(self):
+        return f"{self.id} ({self.codename})"
+
     @hybrid_property
     def name(self):
         return self.codename

portal/rest/projects.py

@@ -21,7 +21,6 @@ log = logger.get_logger(__name__)
 @projects_namespace.param('cid', 'Course id')
 @projects_namespace.response(404, 'Course not found')
 class ProjectsList(Resource):
-
     @jwt_required
     @projects_namespace.response(404, 'Course not found')
     # @projects_namespace.response(200, 'Projects list', model=projects_schema)

portal/rest/schemas.py

@@ -265,6 +265,7 @@ class SubmissionSchema(BaseSchema, Schema):
     scheduled_for = fields.LocalDateTime()
     parameters = fields.Dict()
     project = NESTED['project']
+    course = NESTED['course']
     user = NESTED['user']
 
 

portal/service/courses.py

@@ -38,7 +38,7 @@ class CourseService:
         course = Course()
         self._course = course
         _set_course_props(course=course, data=data)
-        log.debug(f"[CREATE] Course {course.id}: {course}")
+        log.debug(f"[CREATE] Course {course.log_name}: {course}")
         return course
 
     def copy_course(self, target: Course, config: dict) -> Course:
@@ -62,14 +62,14 @@ class CourseService:
                 ProjectService(project).copy_project(target)
 
         general.write_entity(target)
-        log.info(f"[IMPORT] From {self.course.id} to:  {target.id}.")
+        log.info(f"[IMPORT] From {self.course.log_name} to {target.log_name}.")
 
         return target
 
     def delete_course(self):
         """Deletes course
         """
-        log.info(f"[DELETE] Course: {self.course.id} ({self.course.codename})")
+        log.info(f"[DELETE] Course: {self.course.log_name}")
         general.delete_entity(self.course)
 
     def update_course(self, data: dict) -> Course:
@@ -81,7 +81,7 @@ class CourseService:
         Returns(Course): Updated course instance
         """
         _set_course_props(course=self.course, data=data)
-        log.info(f"[UPDATE] Course {self.course.id} ({self.course.codename}): {self.course}.")
+        log.info(f"[UPDATE] Course {self.course.log_name}: {self.course}.")
         return self.course
 
     def update_notes_token(self, token: str) -> Course:
@@ -94,8 +94,7 @@ class CourseService:
         """
         self.course.notes_access_token = token
         general.write_entity(self.course)
-        log.info(f"[UPDATE] Notes access token "
-                 f"({self.course.codename}) [{self.course.id}]: {token}")
+        log.info(f"[UPDATE] Notes access token {self.course.log_name}: {token}")
         return self.course
 
     def find_all_courses(self) -> List[Course]:

portal/service/emails.py

@@ -3,6 +3,7 @@ Emails service
 """
 
 import logging
+
 from emails.template import JinjaTemplate as T
 from flask_emails import Message
 
@@ -142,8 +143,7 @@ def send_message_to_user(user: User = None, message: Message = None,
     email = user.email
     to = (user.name, email)
     if email and message:
-        log.debug(f"[EMAIL] Send message to user({user.id}[{user.username}] :: {email}): "
-                  f"{message}")
+        log.debug(f"[EMAIL] Send message to user [{user.log_name} :: {email}]: {message}")
         message.send(to=to, render=context)
     return message
 

portal/service/groups.py

@@ -36,8 +36,8 @@ class GroupService:
         Returns: the new group
         """
         source = self.group
-        log.info(
-            f"[COPY] Group: {source.id} to course {target.id} with config: {with_users}")
+        log.info(f"[COPY] Group: {source.log_name} to course {target.log_name} "
+                 f"with config: {with_users}")
         new_name = get_new_name(source, target)
         new_group = Group(target, codename=new_name)
         new_group.description = source.description
@@ -50,7 +50,7 @@ class GroupService:
     def delete_group(self):
         """Deletes group
         """
-        log.info(f"[DELETE] Group {self.group.id} ({self.group.codename})")
+        log.info(f"[DELETE] Group {self.group.log_name}")
         general.delete_entity(self.group)
 
     def update_group(self, data: dict) -> Group:
@@ -61,7 +61,7 @@ class GroupService:
 
         Returns(Group): Updated group instance
         """
-        log.info(f"[UPDATE] Group {self.group.codename} to {self.group}.")
+        log.info(f"[UPDATE] Group {self.group.log_name}: {self.group}.")
         return _set_group_data(self.group, data)
 
     def create_group(self, course: Course, **data) -> Course:
@@ -81,7 +81,7 @@ class GroupService:
         self._group = new_group
         _set_group_data(new_group, data)
         log.info(
-            f"[CREATE] Group for course {course.id} ({course.codename}): {new_group}")
+            f"[CREATE] New group {new_group.log_name}: {new_group}")
         return new_group
 
     def update_membership(self, data: dict):
@@ -102,7 +102,7 @@ class GroupService:
             users = users_old.difference(to_remove)
         self.group.users = list(users)
         general.write_entity(self.group)
-        log.info(f"[UPDATE] Users of group {self.group.codename} to {self.group.users}.")
+        log.info(f"[UPDATE] Users of group {self.group.log_name} to {self.group.users}.")
 
     def find_users_by_role(self, course: Course, role_id=None) -> List[User]:
         """Finds users in the group by role
@@ -131,11 +131,10 @@ class GroupService:
             self.group.users.append(user)
             general.write_entity(self.group)
             log.info(
-                f"[ADD] Added user {user.username} to group {self.group.codename} "
-                f"in course {course.codename}.")
+                f"[ADD] Added user {user.log_name} to group {self.group.log_name}")
         else:
-            log.info(f"[ADD] User {user.username} is already in group {self.group.codename} "
-                     f"in course {course.codename}: no change.")
+            log.info(f"[ADD] User {user.log_name} is already in group {self.group.log_name}: "
+                     f"no change.")
         return self.group
 
     def remove_user(self, user: User) -> Group:
@@ -152,11 +151,10 @@ class GroupService:
             general.write_entity(self.group)
         except ValueError:
             message = f"[REMOVE] Could not remove user " \
-                      f"{user.username} from group {self.group.codename} in course " \
-                      f"{course.codename}: group does not contain user."
+                      f"{user.log_name} from group {self.group.log_name}: " \
+                      f"group does not contain user."
             raise PortalAPIError(400, message=message)
-        log.info(f"[REMOVE] User {user.username} from group {self.group.codename}"
-                 f" in course {course.codename}.")
+        log.info(f"[REMOVE] User {user.log_name} from group {self.group.log_name}")
         return self.group
 
     def import_group(self, data: dict, course: Course = None) -> Group:
@@ -172,10 +170,8 @@ class GroupService:
         self._group = source_group
         new_group = self.copy_group(course, data['with_users'])
         general.write_entity(new_group)
-        log.info(
-            f"[IMPORT] Group {source_group.name} from course "
-            f"{source_course.codename} to course {course.codename} "
-            f"as {new_group.codename}.")
+        log.info(f"[IMPORT] Group {source_group.log_name} to course {course.log_name}"
+                 f" as {new_group.log_name}.")
         return new_group
 
     def list_groups(self, course: Course) -> List[Group]:
@@ -206,11 +202,10 @@ class GroupService:
             general.write_entity(self.group)
         except ValueError:
             message = f"[REMOVE] Could not remove project " \
-                      f"{project.codename} from group {self.group.codename} in course " \
-                      f"{course.codename}: group does not contain project."
+                      f"{project.log_name} from group {self.group.log_name}: " \
+                      f"group does not contain project."
             raise PortalAPIError(400, message=message)
-        log.info(f"[REMOVE] Project {project.codename} from group {self.group.codename}"
-                 f" in course {course.codename}.")
+        log.info(f"[REMOVE] Project {project.log_name} from group {self.group.log_name}")
         return self.group
 
     def add_project(self, project: Project) -> Group:
@@ -224,12 +219,10 @@ class GroupService:
         if project not in self.group.projects:
             self.group.projects.append(project)
             general.write_entity(self.group)
-            log.info(
-                f"[ADD] Added project {project.codename} to group "
-                f"{self.group.codename} in course {course.codename}.")
+            log.info(f"[ADD] Added project {project.log_name} to group {self.group.log_name}")
         else:
-            log.info(f"[ADD] User {project.codename} is already in group {self.group.codename} "
-                     f"in course {course.codename}: no change.")
+            log.info(f"[ADD] Project {project.log_name} is already in group {self.group.log_name}: "
+                     f"no change.")
         return self.group
 
     def find_projects(self) -> List[Project]:

portal/service/permissions.py

@@ -36,8 +36,8 @@ class PermissionServiceCheck:
 
     def permissions(self, permissions) -> bool:
         client = self.service.client
-        log.debug(f"[PERM] Client {client.id} ({client.codename}) in course "
-                  f"\"{self.service.course.codename}\": {permissions}")
+        log.debug(f"[PERM] Client {client.log_name} in course "
+                  f"\"{self.service.course.log_name}\": {permissions}")
         if self.sysadmin():
             return True
 
@@ -48,7 +48,7 @@ class PermissionServiceCheck:
         res = any(_resolve_permissions(conjunction, effective_permissions) for conjunction in
                   permissions)
 
-        log.debug(f"[PERM] {permissions} for {client.id} ({client.codename}): {res}")
+        log.debug(f"[PERM] {permissions} for {client.log_name}: {res}")
         return res
 
     def any_check(self, *params) -> bool:
@@ -229,8 +229,8 @@ class PermissionsService:
                 if not key.startswith("_") and key not in FILTER_PERMISSION_ATTRS:
                     result[key] = result.get(key) or value
         log.debug(
-            f"[PERM] Effective permissions: {self.client.id} "
-            f"in course {course.codename}: {result}")
+            f"[PERM] Effective permissions: {self.client.log_name} "
+            f"in course {course.log_name}: {result}")
         return result
 
     def submission_access_group(self, submission, perm):

portal/service/projects.py

@@ -68,7 +68,7 @@ class ProjectService:
     def delete_project(self):
         """Deletes a project
         """
-        log.info(f"[DELETE] Project {self.project.id} ({self.project.codename}).")
+        log.info(f"[DELETE] Project {self.project.log_name}.")
         general.delete_entity(self.project)
 
     def update_project(self, data: dict) -> Project:
@@ -79,7 +79,7 @@ class ProjectService:
 
         Returns(Project): Updated project
         """
-        log.info(f"[UPDATE] Project {self.project.id} ({self.project.codename}): {data}.")
+        log.info(f"[UPDATE] Project {self.project.log_name}: {data}.")
         return _set_project_data(self.project, data)
 
     def create_project(self, course: Course, data: dict) -> Project:
@@ -106,11 +106,9 @@ class ProjectService:
 
         Returns(ProjectConfig): Project config instance
         """
-        course = self.project.course
         self.project.set_config(**data)
         general.write_entity(self.project)
-        log.info(f"[UPDATE] Configuration for project {self.project.id} in '"
-                 f"course {course.id} to {data}.")
+        log.info(f"[UPDATE] Configuration for project {self.project.log_name} in {data}.")
         return self.project.config
 
     def query_submissions(self, user: User) -> BaseQuery:

portal/service/reviews.py

@@ -37,7 +37,7 @@ class ReviewService:
             ReviewItem(user=author, review=self.review, file=item['file'], line=item['line'],
                        content=item['content'])
         log.info(f"Added review items {items} to review {self.review.id} "
-                 f"for submission {self.submission.id}")
+                 f"for submission {self.submission.log_name}")
         write_entity(self.review)
         return self.review
 
@@ -48,7 +48,7 @@ class ReviewService:
 
         Returns(Review): Review instance
         """
-        log.debug(f"[CREATE] Review for submission: {self.submission.id}")
+        log.debug(f"[CREATE] Review for submission: {self.submission.log_name}")
         review = Review(submission=self.submission)
         write_entity(review)
         return review

portal/service/roles.py

@@ -36,8 +36,7 @@ class RoleService:
         Returns(Role): Copied role
         """
         source = self.role
-        log.info(f"[COPY] Role: {self.role.id} to course {target.id} "
-                 f"with config: {with_clients}")
+        log.info(f"[COPY] Role: {self.role.log_name} with config: {with_clients}")
         new_role = self.__copy_new_role(target)
         if with_clients == "with_users":
             for clients in source.clients:
@@ -54,7 +53,7 @@ class RoleService:
     def delete_role(self):
         """Deletes role for the DB
         """
-        log.info(f"[DELETE] Role: {self.role.id} ({self.role.name})")
+        log.info(f"[DELETE] Role: {self.role.log_name}")
         delete_entity(self.role)
 
     def update_role(self, data: dict) -> Role:
@@ -63,7 +62,7 @@ class RoleService:
             data(dict): Data for role update
         Returns(Role): Updated role
         """
-        log.info(f"[UPDATE] Role {self.role.id} to {self.role}.")