Find all refactor - do not list archived projects/courses

management/data/__init__.py

@@ -13,7 +13,8 @@ from management.data.data_dev import init_dev_data
 from management.data.data_prod import init_prod_data
 from management.data.data_test import init_test_data
 from portal import logger
-from portal.database.models import Course, Role, Secret, Submission, SubmissionState, User
+from portal.database.models import Course, Role, Secret, Submission, User
+from portal.database import SubmissionState
 from portal.service.find import FindService
 from portal.service.users import UserService
 from portal.tools import time

management/data/data_dev.py

@@ -4,7 +4,8 @@ from flask import Flask
 from flask_sqlalchemy import SQLAlchemy
 
 from management.data.shared import DataFactory
-from portal.database.models import Course, Review, Secret, Submission, SubmissionState
+from portal.database.models import Course, Review, Secret, Submission
+from portal.database import SubmissionState
 from portal.tools import time
 
 

management/data/data_test.py

@@ -4,7 +4,8 @@ from flask import Flask
 from flask_sqlalchemy import SQLAlchemy
 
 from management.data.shared import DataFactory
-from portal.database.models import Review, Secret, SubmissionState
+from portal.database.models import Review, Secret
+from portal.database import SubmissionState
 from portal.tools import time
 
 

management/data/shared.py

@@ -123,7 +123,7 @@ class DataFactory:
     def __create_entity(self, klass, *args, **kwargs):
         entity = klass(*args, **kwargs)
         self.session.add(entity)
-        log.info(f"[SAMPLE] Create {entity}")
+        log.info(f"[SAMPLE] Create {klass.__name__}: {entity}")
         return entity
 
     def create_worker(self, name: str, url: str) -> Worker:

migrations/versions/61787ca3cab6_.py

+"""empty message
+
+Revision ID: 61787ca3cab6
+Revises: b7f7f426d205
+Create Date: 2019-04-14 12:03:36.303527
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+import portal.database.types
+import portal.database.enums as enums
+
+# revision identifiers, used by Alembic.
+revision = '61787ca3cab6'
+down_revision = 'b7f7f426d205'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('course', sa.Column('state', portal.database.types.CustomEnum(enums.CourseState),
+                                      nullable=True))
+    op.add_column('user', sa.Column('gitlab_username', sa.String(length=50), nullable=True))
+    op.add_column('user', sa.Column('managed', sa.Boolean(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('user', 'managed')
+    op.drop_column('user', 'gitlab_username')
+    op.drop_column('course', 'state')
+    # ### end Alembic commands ###

portal/async_celery/submission_processor.py

@@ -5,7 +5,7 @@ from typing import Optional
 
 from portal import logger
 from portal.database import Project, Submission, SubmissionState, Worker
-from portal.database.models import WorkerState
+from portal.database.enums import WorkerState
 from portal.service import errors
 from portal.storage import UploadedEntity, entities
 

portal/database/__init__.py

@@ -2,7 +2,8 @@
 Database layer module
 """
 
-from .models import User, Group, Project, ProjectState, ProjectConfig, Role, Course, Worker,\
-    Submission, SubmissionState, Review, ReviewItem
+from .models import User, Group, Project, ProjectConfig, Role, Course, Worker,\
+    Submission, Review, ReviewItem
+from portal.database.enums import ProjectState, SubmissionState
 
 from .exceptions import PortalDbError

portal/database/enums.py

+import enum
+
+
+class ClientType(enum.Enum):
+    """All known client types
+    """
+    USER = 'user'
+    WORKER = 'worker'
+
+
+class CourseState(enum.Enum):
+    """All the states in which the project can be
+    """
+    ACTIVE = 'active'
+    INACTIVE = 'inactive'
+    ARCHIVED = 'archived'
+
+
+class ProjectState(enum.Enum):
+    """All the states in which the project can be
+    """
+    ACTIVE = 1
+    INACTIVE = 2
+    ARCHIVED = 3
+
+
+class SubmissionState(enum.Enum):
+    """Enum of the submissions states
+    """
+    CREATED = 1
+    READY = 2
+    QUEUED = 3
+    IN_PROGRESS = 4
+    FINISHED = 5
+    CANCELLED = 6
+    ABORTED = 7
+    ARCHIVED = 8
+
+
+class WorkerState(enum.Enum):
+    """Possible worker states
+    """
+    CREATED = 'created'
+    READY = 'ready'
+    STOPPED = 'stopped'

portal/database/mixins.py

 """
 A collection of Mixins specifying common behaviour and attributes of database entities.
 """
-from typing import List
+from typing import Dict, List
 
 from sqlalchemy.ext.hybrid import hybrid_property
 
@@ -32,14 +32,8 @@ def _str(instance) -> str:
     return _repr(instance=instance)
 
 
-class EntityBase:
-    """Entity mixin for the models
-
-    Class attributes:
-        created_at(Column): Date when the entity has been created
-        updated_at(Column): Date when the entity has been updated
-    """
-    BASE_PARAMS = ['created_at', 'updated_at']
+class MetaModelBase:
+    BASE_PARAMS = []
     LISTABLE = []
     UPDATABLE = []
 
@@ -55,27 +49,65 @@ class EntityBase:
     def listable_params(cls) -> List[str]:
         return bound_update_class_var(cls, 'LISTABLE')
 
-    created_at = db.Column(db.TIMESTAMP, default=db.func.now())
-    updated_at = db.Column(db.TIMESTAMP, default=db.func.now(), onupdate=db.func.now())
-
     def __repr__(self):
         return _repr(self)
 
     def __str__(self):
         return _str(self)
 
+    def __getitem__(self, item):
+        return getattr(self, item)
+
+    @property
+    def log_name(self) -> str:
+        name = ""
+        if hasattr(self, 'namespace'):
+            name = f"[{self.namespace}]"
+        if hasattr(self, 'id') and self.id is not None:
+            name = f"{name} ({self.id})"
+        if name == "":
+            return str(self.serialize)
+        return name
+
+    @property
+    def serialize(self) -> Dict:
+        """Return object data in easily serializeable format"""
+        serialized_obj = {}
+        if not hasattr(self, '__table__'):
+            return self.__dict__
+        for column in self.__table__.columns:
+            serialized_obj[column.key] = self[column.key]
+        return serialized_obj
+
+
+class EntityBase(MetaModelBase):
+    """Entity mixin for the models
+
+    Class attributes:
+        created_at(Column): Date when the entity has been created
+        updated_at(Column): Date when the entity has been updated
+    """
+    BASE_PARAMS = ['created_at', 'updated_at']
+    LISTABLE = []
+    UPDATABLE = []
+
+    created_at = db.Column(db.TIMESTAMP, default=db.func.now())
+    updated_at = db.Column(db.TIMESTAMP, default=db.func.now(), onupdate=db.func.now())
 
-class NamedMixin:
-    LISTABLE = ['name', 'codename']
-    UPDATABLE = [*LISTABLE, 'description']
-    BASE_PARAMS = [*UPDATABLE]
 
-    _name = db.Column('name', db.String(50), nullable=False)
+class CodeNameMixin:
+    LISTABLE = ['codename', 'namespace']
+    UPDATABLE = ['codename']
+    BASE_PARAMS = ['namespace', *UPDATABLE]
+
     _codename = db.Column('codename', db.String(30), nullable=False)
-    description = db.Column(db.Text)
 
     @hybrid_property
-    def codename(self):
+    def namespace(self) -> str:
+        return self.codename
+
+    @hybrid_property
+    def codename(self) -> str:
         return self._codename
 
     @codename.setter
@@ -89,10 +121,20 @@ class NamedMixin:
         Args:
             value: Codename of the entity
         """
-        if value:
-            self._codename = sanitize_code_name(value)
-            if self._name is None:
-                self._name = self._codename
+        if value is None:
+            return
+        self._codename = sanitize_code_name(value)
+        if hasattr(self, '_name') and self._name is None:
+            self._name = self._codename
+
+
+class NamedMixin(CodeNameMixin):
+    LISTABLE = ['name', 'codename', 'namespace']
+    UPDATABLE = ['description', 'name', 'codename']
+    BASE_PARAMS = ['namespace', *UPDATABLE]
+
+    _name = db.Column('name', db.String(50), nullable=False)
+    description = db.Column(db.Text)
 
     @hybrid_property
     def name(self) -> str:

portal/database/models.py

@@ -2,7 +2,6 @@
 Models module where all of the models are specified
 """
 
-import enum
 import logging
 import uuid
 from typing import List
@@ -16,9 +15,11 @@ from sqlalchemy_continuum import make_versioned
 from werkzeug.security import check_password_hash, generate_password_hash
 
 from portal import db
+from portal.database.enums import ClientType, CourseState, ProjectState, SubmissionState, \
+    WorkerState
 from portal.database.exceptions import PortalDbError
-from portal.database.mixins import EntityBase, NamedMixin
-from portal.database.types import JSONEncodedDict, YAMLEncodedDict
+from portal.database.mixins import CodeNameMixin, EntityBase, MetaModelBase, NamedMixin
+from portal.database.types import CustomEnum, JSONEncodedDict, YAMLEncodedDict
 from portal.tools import time
 from portal.tools.time import normalize_time
 
@@ -26,14 +27,7 @@ make_versioned(user_cls=None)
 log = logging.getLogger(__name__)
 
 
-class ClientType(enum.Enum):
-    """All known client types
-    """
-    USER = 'user'
-    WORKER = 'worker'
-
-
-class Client(db.Model, EntityBase):
+class Client(db.Model, MetaModelBase, CodeNameMixin):
     """Client entity model
     """
     __tablename__ = 'client'
@@ -41,7 +35,6 @@ class Client(db.Model, EntityBase):
     UPDATABLE = ['codename']
     id = db.Column(db.String(length=36), default=lambda: str(uuid.uuid4()), primary_key=True)
     type = db.Column(db.Enum(ClientType, name='ClientType'), nullable=False)
-    codename = db.Column(db.String(30), unique=True, nullable=False)
     secrets = db.relationship('Secret', back_populates='client', uselist=True,
                               cascade="all, delete-orphan", passive_deletes=True)
 
@@ -50,10 +43,6 @@ class Client(db.Model, EntityBase):
         'polymorphic_on': type
     }
 
-    @property
-    def log_name(self):
-        return f"{self.id} ({self.codename})"
-
     def __init__(self, client_type: ClientType):
         self.type = client_type
 
@@ -140,9 +129,9 @@ class Secret(db.Model, EntityBase):
         self.value = generate_password_hash(value)
         self.expires_at = expires_at
 
-    @property
-    def log_name(self):
-        return f"{self.id} ({self.name})"
+    @hybrid_property
+    def namespace(self) -> str:
+        return f'{self.client.codename}/{self.name}'
 
     @property
     def expired(self) -> bool:
@@ -157,7 +146,7 @@ class Secret(db.Model, EntityBase):
         return time.normalize_time(self.expires_at) < time.current_time()
 
 
-class User(Client):
+class User(Client, EntityBase):
     """User entity model
 
     Attributes:
@@ -315,27 +304,17 @@ def _get_class_based_on_client_type(client_type):
     return klass
 
 
-class CourseState(enum.Enum):
-    """All the states in which the project can be
-    """
-    ACTIVE = 1
-    INACTIVE = 2
-    ARCHIVED = 3
-
-
 class Course(db.Model, EntityBase, NamedMixin):
     """Course model
     """
     UPDATABLE = ['faculty_id']
-    BASE_PARAMS = ['id', *UPDATABLE, 'namespace']
+    BASE_PARAMS = ['id', *UPDATABLE]
     LISTABLE = [*BASE_PARAMS]
     __tablename__ = 'course'
     id = db.Column(db.String(length=36), default=lambda: str(uuid.uuid4()), primary_key=True)
     notes_access_token = db.Column(db.String(256))
     faculty_id = db.Column(db.Integer)
-
-    state = db.Column(db.Enum(CourseState, name='CourseState'), nullable=False,
-                      default=CourseState.ACTIVE)
+    state = db.Column(CustomEnum(CourseState))
 
     roles = db.relationship("Role", back_populates="course", cascade="all, delete-orphan",
                             passive_deletes=True)
@@ -348,21 +327,15 @@ class Course(db.Model, EntityBase, NamedMixin):
         db.UniqueConstraint('codename', name='course_unique_codename'),
     )
 
-    @hybrid_property
-    def namespace(self) -> str:
-        return self.codename
-
-    @property
-    def log_name(self):
-        return f"{self.id} ({self.namespace})"
-
     def is_api_enabled(self) -> bool:
         return self.notes_access_token is not None and self.notes_access_token != ""
 
     def __init__(self, name: str = None, codename: str = None, notes_access_token: str = None,
-                 description: str = None, faculty_id: int = None) -> None:
+                 description: str = None, faculty_id: int = None,
+                 state: CourseState = CourseState.INACTIVE) -> None:
         """Creates course instance
         Args:
+            state (CourseState): State of the course
             description (str): Course's description
             name(str): Course name
             codename(str): Course codename
@@ -374,6 +347,7 @@ class Course(db.Model, EntityBase, NamedMixin):
         self.description = description
         self.notes_access_token = notes_access_token
         self.faculty_id = faculty_id
+        self.state = state
 
     def __eq__(self, other):
         return self.id == other.id
@@ -415,14 +389,6 @@ class Course(db.Model, EntityBase, NamedMixin):
                                                roles=roles, client_type=client_type).all()
 
 
-class ProjectState(enum.Enum):
-    """All the states in which the project can be
-    """
-    ACTIVE = 1
-    INACTIVE = 2
-    ARCHIVED = 3
-
-
 class Project(db.Model, EntityBase, NamedMixin):
     """Project model
     Class Attributes:
@@ -435,8 +401,8 @@ class Project(db.Model, EntityBase, NamedMixin):
         submissions: list of submissions associated with the project
     """
     UPDATABLE = ['assignment_url', 'submit_instructions', 'submit_configurable']
-    BASE_PARAMS = ['id', *UPDATABLE, 'namespace']
-    LISTABLE = ['id', 'assignment_url', 'namespace', 'submit_configurable']
+    BASE_PARAMS = ['id', *UPDATABLE]
+    LISTABLE = ['id', 'assignment_url', 'submit_configurable']
 
     __tablename__ = 'project'
     id = db.Column(db.String(length=36), default=lambda: str(uuid.uuid4()), primary_key=True)
@@ -461,16 +427,16 @@ class Project(db.Model, EntityBase, NamedMixin):
     def namespace(self) -> str:
         return f"{self.course.codename}/{self.codename}"
 
-    @property
-    def log_name(self) -> str:
-        return f"{self.id} ({self.namespace})"
+    @hybrid_property
+    def state(self) -> ProjectState:
+        return self.get_state_by_timestamp()
 
-    def state(self, timestamp=time.current_time()) -> ProjectState:
+    def get_state_by_timestamp(self, timestamp=time.current_time()) -> ProjectState:
         """Gets project state based on the timestamp
-        Args:
-            timestamp: Time for which the state should be calculated
-        Returns:
-        """
+               Args:
+                   timestamp: Time for which the state should be calculated
+               Returns:
+               """
         if not (self.config.submissions_allowed_from or
                 self.config.submissions_allowed_to or
                 self.config.archive_from):
@@ -560,7 +526,7 @@ class ProjectConfig(db.Model, EntityBase):
                  'pre_submit_script', 'post_submit_script', 'submission_scheduler_config',
                  'submissions_allowed_from', 'submissions_allowed_to', 'archive_from']
     BASE_PARAMS = ['id', *UPDATABLE]
-    LISTABLE = ['id', 'assignment_url', 'namespace']
+    LISTABLE = ['id', 'assignment_url']
     __tablename__ = 'projectConfig'
     id = db.Column(db.String(length=36), default=lambda: str(
         uuid.uuid4()), primary_key=True)
@@ -694,9 +660,9 @@ class Role(db.Model, EntityBase, NamedMixin):
                    'read_submissions_own', 'read_reviews_all', 'read_reviews_groups',
                    'read_reviews_own', 'write_reviews_all', 'write_reviews_group',
                    'write_reviews_own')
-    BASE_PARAMS = [*PERMISSIONS, 'namespace', 'id']
+    BASE_PARAMS = [*PERMISSIONS, 'id']
     UPDATABLE = PERMISSIONS
-    LISTABLE = ['id', 'namespace']
+    LISTABLE = ['id']
 
     __tablename__ = 'role'
     id = db.Column(db.String(length=36), default=lambda: str(
@@ -736,10 +702,6 @@ class Role(db.Model, EntityBase, NamedMixin):
     write_reviews_group = db.Column(db.Boolean, default=False, nullable=False)
     write_reviews_own = db.Column(db.Boolean, default=False, nullable=False)
 
-    @property
-    def log_name(self):
-        return f"{self.id} ({self.course.codename}/{self.codename})"
-
     def set_permissions(self, **kwargs):
         """Sets permissions for the role
         Args:
@@ -784,7 +746,7 @@ class Group(db.Model, EntityBase, NamedMixin):
         projects: Collection of projects that are allowed for the Group
     """
     __tablename__ = 'group'
-    LISTABLE = ['id', 'namespace']
+    LISTABLE = ['id']
     BASE_PARAMS = LISTABLE
     id = db.Column(db.String(length=36), default=lambda: str(
         uuid.uuid4()), primary_key=True)
@@ -803,10 +765,6 @@ class Group(db.Model, EntityBase, NamedMixin):
     def namespace(self) -> str:
         return f"{self.course.codename}/{self.codename}"
 
-    @property
-    def log_name(self):
-        return f"{self.id} ({self.namespace})"
-
     def __init__(self, course: Course, name: str = None,
                  description: str = None, codename: str = None):
         """Creates instance of the group
@@ -831,19 +789,6 @@ class Group(db.Model, EntityBase, NamedMixin):
         return queries.group_users(self, role).all()
 
 
-class SubmissionState(enum.Enum):
-    """Enum of the submissions states
-    """
-    CREATED = 1
-    READY = 2
-    QUEUED = 3
-    IN_PROGRESS = 4
-    FINISHED = 5
-    CANCELLED = 6
-    ABORTED = 7
-    ARCHIVED = 8
-
-
 class Submission(db.Model, EntityBase):
     """Submission model
     Class Attributes:
@@ -858,7 +803,7 @@ class Submission(db.Model, EntityBase):
         project: Associated project for the submission
         review: Review associated with the submission
     """
-    LISTABLE = ['id', 'state', 'points', 'result', 'scheduled_for', 'namespace']
+    LISTABLE = ['id', 'state', 'points', 'result', 'scheduled_for']
     BASE_PARAMS = ['parameters', 'source_hash', *LISTABLE]
     __tablename__ = 'submission'
     id = db.Column(db.String(length=36), default=lambda: str(
@@ -903,11 +848,6 @@ class Submission(db.Model, EntityBase):
     def namespace(self) -> str:
         return f"{self.project.namespace}/{self.user.codename}/{self.created_at}"
 
-    @property
-    def log_name(self):
-        return f"{self.id} ({self.course.codename}/{self.project.codename} for " \
-            f"{self.user.username})"
-
     def change_state(self, new_state):
         # open to extension (state transition validation, ...)
         if new_state in Submission.ALLOWED_TRANSITIONS.keys():
@@ -1057,15 +997,7 @@ class ReviewItem(db.Model, EntityBase):
         return self.id == other.id
 
 
-class WorkerState(enum.Enum):
-    """Possible worker states
-    """
-    CREATED = 'created'
-    READY = 'ready'
-    STOPPED = 'stopped'
-
-
-class Worker(Client):
+class Worker(Client, EntityBase):
     """Worker model:
     Class Attributes:
         id: UUID of the submission
@@ -1090,9 +1022,9 @@ class Worker(Client):
         'polymorphic_identity': ClientType.WORKER,
     }
 
-    @property
-    def log_name(self):
-        return f"{self.id} ({self.codename})"
+    @hybrid_property
+    def namespace(self) -> str:
+        return self.codename
 
     @hybrid_property
     def name(self):

portal/database/queries.py

@@ -5,8 +5,10 @@ from flask_sqlalchemy import BaseQuery
 from sqlalchemy import func
 
 from portal import db
-from portal.database.models import Client, ClientType, Course, Group, Project, Role, Submission, \
-    SubmissionState, User, Worker
+from portal.database.models import Client, Course, Group, Project, Role, Submission, \
+    User, Worker
+from portal.database import SubmissionState
+from portal.database.enums import ClientType
 
 log = logging.getLogger(__name__)
 
@@ -204,7 +206,7 @@ def list_submissions_for_user(client: User = None, user_ids=None,
     if client is not None and not client.is_admin:
         query = _filter_by_client(query, client)
 
-    log.debug(f"[QUERY] find submissions: {query}")
+    log.trace(f"[QUERY] find submissions: {query}")
     return query
 
 

portal/database/types.py

 import json
+from enum import Enum
 
 import sqlalchemy
 import yaml
@@ -35,3 +36,21 @@ class YAMLEncodedDict(TypeDecorator):
         if value is not None: