Update to move dirs

app.py

@@ -3,6 +3,8 @@ Main application module
 - Creates instance of the flask app named app
 - registers commands
 """
+from pathlib import Path
+
 from portal.logger import Logging
 
 Logging().load_config()
@@ -54,7 +56,7 @@ def cli_mgmt_arch_submissions():
 
 
 @management_cli.command('shell')
-def cli_mgmt_arch_submissions():
+def cli_mgmg_shell():
     print("[MGMT] Starting interactive shell")
     with app.app_context():
         import IPython
@@ -64,6 +66,20 @@ def cli_mgmt_arch_submissions():
         IPython.embed()
 
 
+@management_cli.command('load-script', help='DANGER ZONE - load script and '
+                                            'execute it with app context')
+@click.argument('file')
+def cli_mgmt_load_script(file):
+    print("[MGMT] loading script")
+    with app.app_context():
+        fp = Path(file).absolute()
+        if fp.exists():
+            content = fp.read_text('utf-8')
+            exec(content)
+        else:
+            print(f"[ERR] File not exists: {fp}")
+
+
 @management_cli.command('delete-cancelled')
 def cli_mgmt_arch_submissions():
     print("[MGMT] Deleting cancelled submissions - NOW")

hack/updates/transform_storage_dirs.py

+import shutil
+from pathlib import Path
+
+from portal import storage_wrapper
+from portal.database import models
+
+UPDATE_NAME = "transform_storage_dirs"
+DEPENDS_ON = []
+
+
+def change_dirs(entity, base_path: Path):
+    old_path = base_path / entity.id
+    new_path = base_path / entity.storage_dirname
+    if old_path.exists():
+        print(f"[MOVE] FROM {old_path} to {new_path}")
+        shutil.move(old_path, new_path)
+
+
+def change_project_dirs():
+    print("[UPDATE] Processing projects")
+    tf_path = storage_wrapper.test_files.path
+    for project in models.Project.query.all():
+        change_dirs(project, tf_path)
+
+
+def change_submission_dirs():
+    print("[UPDATE] Processing submissions")
+    source_path = storage_wrapper.submissions.path
+    results_path = storage_wrapper.results.path
+    for submission in models.Submission.query.all():
+        change_dirs(submission, source_path)
+        change_dirs(submission, results_path)
+
+
+change_project_dirs()
+change_submission_dirs()

portal/async_celery/submission_processor.py

@@ -75,12 +75,12 @@ class SubmissionProcessor:
         log.info(f"[ASYNC] Uploading submission: {self.submission.log_name} with {file_params}")
 
         updated_entity: UploadedEntity = self.storage. \
-            submissions.create(dirname=self.submission.id, **file_params)
+            submissions.create(dirname=self.submission.storage_dirname, **file_params)
         self.submission_store_ended(version=updated_entity.version)
 
     def clone(self, target):
         log.info(f"[ASYNC] Cloning submission: {self.submission.log_name} to {target.log_name}")
-        self.storage.submissions.clone(self.submission.id, target.id)
+        self.storage.submissions.clone(self.submission.storage_dirname, target.id)
         self.submission_store_ended(version=self.submission.source_hash)
 
     def send_to_worker(self):
@@ -97,7 +97,7 @@ class SubmissionProcessor:
     def upload_result(self, path, file_params):
         log.info(f"[ASYNC] Uploading result for the submission "
                  f"{self.submission.log_name} with {file_params}")
-        self.storage.results.create(dirname=self.submission.id, **file_params)
+        self.storage.results.create(dirname=self.submission.storage_dirname, **file_params)
         Path(path).unlink()
         self.reset_task_id(SubmissionState.FINISHED)
 

portal/async_celery/tasks.py

@@ -42,9 +42,7 @@ def archive_submission(submission_id: str):
 
 @celery_app.task(name='upload-results-to-storage')
 def upload_results_to_storage(new_submission_id: str, path: str):
-    path = str(path)
-    find_service = FindService()
-    new_submission = find_service.submission(new_submission_id)
+    new_submission = FindService().submission(new_submission_id)
     log.info(f"[SUBMIT] Processing results - upload to the storage for "
              f"{new_submission.log_name}: {path}")
     SubmissionsService(new_submission).upload_results_to_storage(path)

portal/database/models.py

@@ -444,12 +444,13 @@ class Project(db.Model, EntityBase, NamedMixin):
     def state(self) -> ProjectState:
         return self.get_state_by_timestamp()
 
-    def get_state_by_timestamp(self, timestamp=time.current_time()) -> ProjectState:
+    def get_state_by_timestamp(self, timestamp=None) -> ProjectState:
         """Gets project state based on the timestamp
                Args:
                    timestamp: Time for which the state should be calculated
                Returns:
-               """
+        """
+        timestamp = timestamp or time.current_time()
         if not (self.config.submissions_allowed_from or
                 self.config.submissions_allowed_to or
                 self.config.archive_from):
@@ -862,7 +863,7 @@ class Submission(db.Model, EntityBase):
         return f"{self.project.namespace}/{self.user.codename}/{self.created_at}"
 
     @hybrid_property
-    def storage_dirname(self):
+    def storage_dirname(self) -> str:
         created = time.simple_fmt(self.created_at)
         return f"{self.user.username}_{created}_{self.course.codename}_{self.project.codename}"
 

portal/facade/submissions_facade.py

 import logging
 
+from werkzeug.datastructures import FileStorage
+
 from portal.async_celery import tasks
 from portal.database import Project, Submission, SubmissionState, User
 from portal.facade.general_facade import GeneralCRUDFacade
@@ -81,10 +83,10 @@ class SubmissionsFacade(GeneralCRUDFacade):
     def upload_results_to_storage(self, submission: Submission):
         log.info(f"[UPLOAD] Uploading results to storage for "
                  f"{submission.log_name} by {self.client_name}")
-        file = self._request.files['file']
+        file: FileStorage = self._request.files['file']
         submission_service: SubmissionsService = self._service(submission)
-        return tasks.upload_results_to_storage.delay(submission.id,
-                                                     submission_service.get_upload_file_path(file))
+        path = submission_service.get_upload_file_path(file)
+        return tasks.upload_results_to_storage.delay(submission.id, path)
 
     def copy_submission(self, source_submission: Submission, **params):
         """Copies a submission. Used at resubmitting

portal/service/submissions.py

@@ -6,6 +6,7 @@ import json
 import logging
 from pathlib import Path
 
+from werkzeug.datastructures import FileStorage
 from werkzeug.utils import secure_filename
 
 from portal import storage_wrapper
@@ -39,10 +40,11 @@ class SubmissionsService(GeneralService):
             SubmissionsService(submission).archive()
 
     @classmethod
-    def get_upload_file_path(cls, file) -> str:
-        upload_file_is_allowed(file)
-        path = upload_files_to_storage(file)
-        return str(path)
+    def get_upload_file_path(cls, file: FileStorage) -> str:
+        if upload_file_is_allowed(file):
+            path = upload_files_to_storage(file)
+            return str(path)
+        return None
 
     def processor(self) -> SubmissionProcessor:
         return SubmissionProcessor(self.submission)
@@ -152,8 +154,7 @@ class SubmissionsService(GeneralService):
         """
         self.processor().revoke_task()
 
-    def upload_results_to_storage(self, file):
-        path = self.get_upload_file_path(file)
+    def upload_results_to_storage(self, path):
         processor = self.processor()
         file_params = dict(source=dict(url=path, type='zip'))
         processor.upload_result(path=path, file_params=file_params)
@@ -224,7 +225,7 @@ def nonempty_intersection(provided: list, required: list):
     return list(set(provided) & set(required))
 
 
-def upload_files_to_storage(file):
+def upload_files_to_storage(file: FileStorage):
     filename = secure_filename(file.filename)
     uploads = storage_wrapper.results.workspace_path / 'uploads'
     if not uploads.exists():