Loading app.py +17 −1 Original line number Diff line number Diff line Loading @@ -3,6 +3,8 @@ Main application module - Creates instance of the flask app named app - registers commands """ from pathlib import Path from portal.logger import Logging Logging().load_config() Loading Loading @@ -54,7 +56,7 @@ def cli_mgmt_arch_submissions(): @management_cli.command('shell') def cli_mgmt_arch_submissions(): def cli_mgmg_shell(): print("[MGMT] Starting interactive shell") with app.app_context(): import IPython Loading @@ -64,6 +66,20 @@ def cli_mgmt_arch_submissions(): IPython.embed() @management_cli.command('load-script', help='DANGER ZONE - load script and ' 'execute it with app context') @click.argument('file') def cli_mgmt_load_script(file): print("[MGMT] loading script") with app.app_context(): fp = Path(file).absolute() if fp.exists(): content = fp.read_text('utf-8') exec(content) else: print(f"[ERR] File not exists: {fp}") @management_cli.command('delete-cancelled') def cli_mgmt_arch_submissions(): print("[MGMT] Deleting cancelled submissions - NOW") Loading hack/updates/transform_storage_dirs.py +36 −0 Original line number Diff line number Diff line import shutil from pathlib import Path from portal import storage_wrapper from portal.database import models UPDATE_NAME = "transform_storage_dirs" DEPENDS_ON = [] def change_dirs(entity, base_path: Path): old_path = base_path / entity.id new_path = base_path / entity.storage_dirname if old_path.exists(): print(f"[MOVE] FROM {old_path} to {new_path}") shutil.move(old_path, new_path) def change_project_dirs(): print("[UPDATE] Processing projects") tf_path = storage_wrapper.test_files.path for project in models.Project.query.all(): change_dirs(project, tf_path) def change_submission_dirs(): print("[UPDATE] Processing submissions") source_path = storage_wrapper.submissions.path results_path = storage_wrapper.results.path for submission in models.Submission.query.all(): change_dirs(submission, source_path) change_dirs(submission, results_path) change_project_dirs() change_submission_dirs() portal/async_celery/submission_processor.py +3 −3 Original line number Diff line number Diff line Loading @@ -75,12 +75,12 @@ class SubmissionProcessor: log.info(f"[ASYNC] Uploading submission: {self.submission.log_name} with {file_params}") updated_entity: UploadedEntity = self.storage. \ submissions.create(dirname=self.submission.id, **file_params) submissions.create(dirname=self.submission.storage_dirname, **file_params) self.submission_store_ended(version=updated_entity.version) def clone(self, target): log.info(f"[ASYNC] Cloning submission: {self.submission.log_name} to {target.log_name}") self.storage.submissions.clone(self.submission.id, target.id) self.storage.submissions.clone(self.submission.storage_dirname, target.id) self.submission_store_ended(version=self.submission.source_hash) def send_to_worker(self): Loading @@ -97,7 +97,7 @@ class SubmissionProcessor: def upload_result(self, path, file_params): log.info(f"[ASYNC] Uploading result for the submission " f"{self.submission.log_name} with {file_params}") self.storage.results.create(dirname=self.submission.id, **file_params) self.storage.results.create(dirname=self.submission.storage_dirname, **file_params) Path(path).unlink() self.reset_task_id(SubmissionState.FINISHED) Loading portal/async_celery/tasks.py +1 −3 Original line number Diff line number Diff line Loading @@ -42,9 +42,7 @@ def archive_submission(submission_id: str): @celery_app.task(name='upload-results-to-storage') def upload_results_to_storage(new_submission_id: str, path: str): path = str(path) find_service = FindService() new_submission = find_service.submission(new_submission_id) new_submission = FindService().submission(new_submission_id) log.info(f"[SUBMIT] Processing results - upload to the storage for " f"{new_submission.log_name}: {path}") SubmissionsService(new_submission).upload_results_to_storage(path) Loading portal/database/models.py +4 −3 Original line number Diff line number Diff line Loading @@ -444,12 +444,13 @@ class Project(db.Model, EntityBase, NamedMixin): def state(self) -> ProjectState: return self.get_state_by_timestamp() def get_state_by_timestamp(self, timestamp=time.current_time()) -> ProjectState: def get_state_by_timestamp(self, timestamp=None) -> ProjectState: """Gets project state based on the timestamp Args: timestamp: Time for which the state should be calculated Returns: """ """ timestamp = timestamp or time.current_time() if not (self.config.submissions_allowed_from or self.config.submissions_allowed_to or self.config.archive_from): Loading Loading @@ -862,7 +863,7 @@ class Submission(db.Model, EntityBase): return f"{self.project.namespace}/{self.user.codename}/{self.created_at}" @hybrid_property def storage_dirname(self): def storage_dirname(self) -> str: created = time.simple_fmt(self.created_at) return f"{self.user.username}_{created}_{self.course.codename}_{self.project.codename}" Loading Loading
app.py +17 −1 Original line number Diff line number Diff line Loading @@ -3,6 +3,8 @@ Main application module - Creates instance of the flask app named app - registers commands """ from pathlib import Path from portal.logger import Logging Logging().load_config() Loading Loading @@ -54,7 +56,7 @@ def cli_mgmt_arch_submissions(): @management_cli.command('shell') def cli_mgmt_arch_submissions(): def cli_mgmg_shell(): print("[MGMT] Starting interactive shell") with app.app_context(): import IPython Loading @@ -64,6 +66,20 @@ def cli_mgmt_arch_submissions(): IPython.embed() @management_cli.command('load-script', help='DANGER ZONE - load script and ' 'execute it with app context') @click.argument('file') def cli_mgmt_load_script(file): print("[MGMT] loading script") with app.app_context(): fp = Path(file).absolute() if fp.exists(): content = fp.read_text('utf-8') exec(content) else: print(f"[ERR] File not exists: {fp}") @management_cli.command('delete-cancelled') def cli_mgmt_arch_submissions(): print("[MGMT] Deleting cancelled submissions - NOW") Loading
hack/updates/transform_storage_dirs.py +36 −0 Original line number Diff line number Diff line import shutil from pathlib import Path from portal import storage_wrapper from portal.database import models UPDATE_NAME = "transform_storage_dirs" DEPENDS_ON = [] def change_dirs(entity, base_path: Path): old_path = base_path / entity.id new_path = base_path / entity.storage_dirname if old_path.exists(): print(f"[MOVE] FROM {old_path} to {new_path}") shutil.move(old_path, new_path) def change_project_dirs(): print("[UPDATE] Processing projects") tf_path = storage_wrapper.test_files.path for project in models.Project.query.all(): change_dirs(project, tf_path) def change_submission_dirs(): print("[UPDATE] Processing submissions") source_path = storage_wrapper.submissions.path results_path = storage_wrapper.results.path for submission in models.Submission.query.all(): change_dirs(submission, source_path) change_dirs(submission, results_path) change_project_dirs() change_submission_dirs()
portal/async_celery/submission_processor.py +3 −3 Original line number Diff line number Diff line Loading @@ -75,12 +75,12 @@ class SubmissionProcessor: log.info(f"[ASYNC] Uploading submission: {self.submission.log_name} with {file_params}") updated_entity: UploadedEntity = self.storage. \ submissions.create(dirname=self.submission.id, **file_params) submissions.create(dirname=self.submission.storage_dirname, **file_params) self.submission_store_ended(version=updated_entity.version) def clone(self, target): log.info(f"[ASYNC] Cloning submission: {self.submission.log_name} to {target.log_name}") self.storage.submissions.clone(self.submission.id, target.id) self.storage.submissions.clone(self.submission.storage_dirname, target.id) self.submission_store_ended(version=self.submission.source_hash) def send_to_worker(self): Loading @@ -97,7 +97,7 @@ class SubmissionProcessor: def upload_result(self, path, file_params): log.info(f"[ASYNC] Uploading result for the submission " f"{self.submission.log_name} with {file_params}") self.storage.results.create(dirname=self.submission.id, **file_params) self.storage.results.create(dirname=self.submission.storage_dirname, **file_params) Path(path).unlink() self.reset_task_id(SubmissionState.FINISHED) Loading
portal/async_celery/tasks.py +1 −3 Original line number Diff line number Diff line Loading @@ -42,9 +42,7 @@ def archive_submission(submission_id: str): @celery_app.task(name='upload-results-to-storage') def upload_results_to_storage(new_submission_id: str, path: str): path = str(path) find_service = FindService() new_submission = find_service.submission(new_submission_id) new_submission = FindService().submission(new_submission_id) log.info(f"[SUBMIT] Processing results - upload to the storage for " f"{new_submission.log_name}: {path}") SubmissionsService(new_submission).upload_results_to_storage(path) Loading
portal/database/models.py +4 −3 Original line number Diff line number Diff line Loading @@ -444,12 +444,13 @@ class Project(db.Model, EntityBase, NamedMixin): def state(self) -> ProjectState: return self.get_state_by_timestamp() def get_state_by_timestamp(self, timestamp=time.current_time()) -> ProjectState: def get_state_by_timestamp(self, timestamp=None) -> ProjectState: """Gets project state based on the timestamp Args: timestamp: Time for which the state should be calculated Returns: """ """ timestamp = timestamp or time.current_time() if not (self.config.submissions_allowed_from or self.config.submissions_allowed_to or self.config.archive_from): Loading Loading @@ -862,7 +863,7 @@ class Submission(db.Model, EntityBase): return f"{self.project.namespace}/{self.user.codename}/{self.created_at}" @hybrid_property def storage_dirname(self): def storage_dirname(self) -> str: created = time.simple_fmt(self.created_at) return f"{self.user.username}_{created}_{self.course.codename}_{self.project.codename}" Loading