Verified Commit ce1d7678 authored by Peter Stanko's avatar Peter Stanko
Browse files

Update to move dirs

parent 500de1b3
Pipeline #31699 passed with stage
in 7 minutes and 20 seconds
...@@ -3,6 +3,8 @@ Main application module ...@@ -3,6 +3,8 @@ Main application module
- Creates instance of the flask app named app - Creates instance of the flask app named app
- registers commands - registers commands
""" """
from pathlib import Path
from portal.logger import Logging from portal.logger import Logging
Logging().load_config() Logging().load_config()
...@@ -54,7 +56,7 @@ def cli_mgmt_arch_submissions(): ...@@ -54,7 +56,7 @@ def cli_mgmt_arch_submissions():
@management_cli.command('shell') @management_cli.command('shell')
def cli_mgmt_arch_submissions(): def cli_mgmg_shell():
print("[MGMT] Starting interactive shell") print("[MGMT] Starting interactive shell")
with app.app_context(): with app.app_context():
import IPython import IPython
...@@ -64,6 +66,20 @@ def cli_mgmt_arch_submissions(): ...@@ -64,6 +66,20 @@ def cli_mgmt_arch_submissions():
IPython.embed() IPython.embed()
@management_cli.command('load-script', help='DANGER ZONE - load script and '
'execute it with app context')
@click.argument('file')
def cli_mgmt_load_script(file):
print("[MGMT] loading script")
with app.app_context():
fp = Path(file).absolute()
if fp.exists():
content = fp.read_text('utf-8')
exec(content)
else:
print(f"[ERR] File not exists: {fp}")
@management_cli.command('delete-cancelled') @management_cli.command('delete-cancelled')
def cli_mgmt_arch_submissions(): def cli_mgmt_arch_submissions():
print("[MGMT] Deleting cancelled submissions - NOW") print("[MGMT] Deleting cancelled submissions - NOW")
......
import shutil
from pathlib import Path
from portal import storage_wrapper
from portal.database import models
UPDATE_NAME = "transform_storage_dirs"
DEPENDS_ON = []
def change_dirs(entity, base_path: Path):
old_path = base_path / entity.id
new_path = base_path / entity.storage_dirname
if old_path.exists():
print(f"[MOVE] FROM {old_path} to {new_path}")
shutil.move(old_path, new_path)
def change_project_dirs():
print("[UPDATE] Processing projects")
tf_path = storage_wrapper.test_files.path
for project in models.Project.query.all():
change_dirs(project, tf_path)
def change_submission_dirs():
print("[UPDATE] Processing submissions")
source_path = storage_wrapper.submissions.path
results_path = storage_wrapper.results.path
for submission in models.Submission.query.all():
change_dirs(submission, source_path)
change_dirs(submission, results_path)
change_project_dirs()
change_submission_dirs()
...@@ -75,12 +75,12 @@ class SubmissionProcessor: ...@@ -75,12 +75,12 @@ class SubmissionProcessor:
log.info(f"[ASYNC] Uploading submission: {self.submission.log_name} with {file_params}") log.info(f"[ASYNC] Uploading submission: {self.submission.log_name} with {file_params}")
updated_entity: UploadedEntity = self.storage. \ updated_entity: UploadedEntity = self.storage. \
submissions.create(dirname=self.submission.id, **file_params) submissions.create(dirname=self.submission.storage_dirname, **file_params)
self.submission_store_ended(version=updated_entity.version) self.submission_store_ended(version=updated_entity.version)
def clone(self, target): def clone(self, target):
log.info(f"[ASYNC] Cloning submission: {self.submission.log_name} to {target.log_name}") log.info(f"[ASYNC] Cloning submission: {self.submission.log_name} to {target.log_name}")
self.storage.submissions.clone(self.submission.id, target.id) self.storage.submissions.clone(self.submission.storage_dirname, target.id)
self.submission_store_ended(version=self.submission.source_hash) self.submission_store_ended(version=self.submission.source_hash)
def send_to_worker(self): def send_to_worker(self):
...@@ -97,7 +97,7 @@ class SubmissionProcessor: ...@@ -97,7 +97,7 @@ class SubmissionProcessor:
def upload_result(self, path, file_params): def upload_result(self, path, file_params):
log.info(f"[ASYNC] Uploading result for the submission " log.info(f"[ASYNC] Uploading result for the submission "
f"{self.submission.log_name} with {file_params}") f"{self.submission.log_name} with {file_params}")
self.storage.results.create(dirname=self.submission.id, **file_params) self.storage.results.create(dirname=self.submission.storage_dirname, **file_params)
Path(path).unlink() Path(path).unlink()
self.reset_task_id(SubmissionState.FINISHED) self.reset_task_id(SubmissionState.FINISHED)
......
...@@ -42,9 +42,7 @@ def archive_submission(submission_id: str): ...@@ -42,9 +42,7 @@ def archive_submission(submission_id: str):
@celery_app.task(name='upload-results-to-storage') @celery_app.task(name='upload-results-to-storage')
def upload_results_to_storage(new_submission_id: str, path: str): def upload_results_to_storage(new_submission_id: str, path: str):
path = str(path) new_submission = FindService().submission(new_submission_id)
find_service = FindService()
new_submission = find_service.submission(new_submission_id)
log.info(f"[SUBMIT] Processing results - upload to the storage for " log.info(f"[SUBMIT] Processing results - upload to the storage for "
f"{new_submission.log_name}: {path}") f"{new_submission.log_name}: {path}")
SubmissionsService(new_submission).upload_results_to_storage(path) SubmissionsService(new_submission).upload_results_to_storage(path)
......
...@@ -444,12 +444,13 @@ class Project(db.Model, EntityBase, NamedMixin): ...@@ -444,12 +444,13 @@ class Project(db.Model, EntityBase, NamedMixin):
def state(self) -> ProjectState: def state(self) -> ProjectState:
return self.get_state_by_timestamp() return self.get_state_by_timestamp()
def get_state_by_timestamp(self, timestamp=time.current_time()) -> ProjectState: def get_state_by_timestamp(self, timestamp=None) -> ProjectState:
"""Gets project state based on the timestamp """Gets project state based on the timestamp
Args: Args:
timestamp: Time for which the state should be calculated timestamp: Time for which the state should be calculated
Returns: Returns:
""" """
timestamp = timestamp or time.current_time()
if not (self.config.submissions_allowed_from or if not (self.config.submissions_allowed_from or
self.config.submissions_allowed_to or self.config.submissions_allowed_to or
self.config.archive_from): self.config.archive_from):
...@@ -862,7 +863,7 @@ class Submission(db.Model, EntityBase): ...@@ -862,7 +863,7 @@ class Submission(db.Model, EntityBase):
return f"{self.project.namespace}/{self.user.codename}/{self.created_at}" return f"{self.project.namespace}/{self.user.codename}/{self.created_at}"
@hybrid_property @hybrid_property
def storage_dirname(self): def storage_dirname(self) -> str:
created = time.simple_fmt(self.created_at) created = time.simple_fmt(self.created_at)
return f"{self.user.username}_{created}_{self.course.codename}_{self.project.codename}" return f"{self.user.username}_{created}_{self.course.codename}_{self.project.codename}"
......
import logging import logging
from werkzeug.datastructures import FileStorage
from portal.async_celery import tasks from portal.async_celery import tasks
from portal.database import Project, Submission, SubmissionState, User from portal.database import Project, Submission, SubmissionState, User
from portal.facade.general_facade import GeneralCRUDFacade from portal.facade.general_facade import GeneralCRUDFacade
...@@ -81,10 +83,10 @@ class SubmissionsFacade(GeneralCRUDFacade): ...@@ -81,10 +83,10 @@ class SubmissionsFacade(GeneralCRUDFacade):
def upload_results_to_storage(self, submission: Submission): def upload_results_to_storage(self, submission: Submission):
log.info(f"[UPLOAD] Uploading results to storage for " log.info(f"[UPLOAD] Uploading results to storage for "
f"{submission.log_name} by {self.client_name}") f"{submission.log_name} by {self.client_name}")
file = self._request.files['file'] file: FileStorage = self._request.files['file']
submission_service: SubmissionsService = self._service(submission) submission_service: SubmissionsService = self._service(submission)
return tasks.upload_results_to_storage.delay(submission.id, path = submission_service.get_upload_file_path(file)
submission_service.get_upload_file_path(file)) return tasks.upload_results_to_storage.delay(submission.id, path)
def copy_submission(self, source_submission: Submission, **params): def copy_submission(self, source_submission: Submission, **params):
"""Copies a submission. Used at resubmitting """Copies a submission. Used at resubmitting
......
...@@ -6,6 +6,7 @@ import json ...@@ -6,6 +6,7 @@ import json
import logging import logging
from pathlib import Path from pathlib import Path
from werkzeug.datastructures import FileStorage
from werkzeug.utils import secure_filename from werkzeug.utils import secure_filename
from portal import storage_wrapper from portal import storage_wrapper
...@@ -39,10 +40,11 @@ class SubmissionsService(GeneralService): ...@@ -39,10 +40,11 @@ class SubmissionsService(GeneralService):
SubmissionsService(submission).archive() SubmissionsService(submission).archive()
@classmethod @classmethod
def get_upload_file_path(cls, file) -> str: def get_upload_file_path(cls, file: FileStorage) -> str:
upload_file_is_allowed(file) if upload_file_is_allowed(file):
path = upload_files_to_storage(file) path = upload_files_to_storage(file)
return str(path) return str(path)
return None
def processor(self) -> SubmissionProcessor: def processor(self) -> SubmissionProcessor:
return SubmissionProcessor(self.submission) return SubmissionProcessor(self.submission)
...@@ -152,8 +154,7 @@ class SubmissionsService(GeneralService): ...@@ -152,8 +154,7 @@ class SubmissionsService(GeneralService):
""" """
self.processor().revoke_task() self.processor().revoke_task()
def upload_results_to_storage(self, file): def upload_results_to_storage(self, path):
path = self.get_upload_file_path(file)
processor = self.processor() processor = self.processor()
file_params = dict(source=dict(url=path, type='zip')) file_params = dict(source=dict(url=path, type='zip'))
processor.upload_result(path=path, file_params=file_params) processor.upload_result(path=path, file_params=file_params)
...@@ -224,7 +225,7 @@ def nonempty_intersection(provided: list, required: list): ...@@ -224,7 +225,7 @@ def nonempty_intersection(provided: list, required: list):
return list(set(provided) & set(required)) return list(set(provided) & set(required))
def upload_files_to_storage(file): def upload_files_to_storage(file: FileStorage):
filename = secure_filename(file.filename) filename = secure_filename(file.filename)
uploads = storage_wrapper.results.workspace_path / 'uploads' uploads = storage_wrapper.results.workspace_path / 'uploads'
if not uploads.exists(): if not uploads.exists():
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment