Verified Commit ce1d7678 authored by Peter Stanko's avatar Peter Stanko
Browse files

Update to move dirs

parent 500de1b3
Pipeline #31699 passed with stage
in 7 minutes and 20 seconds
......@@ -3,6 +3,8 @@ Main application module
- Creates instance of the flask app named app
- registers commands
"""
from pathlib import Path
from portal.logger import Logging
Logging().load_config()
......@@ -54,7 +56,7 @@ def cli_mgmt_arch_submissions():
@management_cli.command('shell')
def cli_mgmt_arch_submissions():
def cli_mgmg_shell():
print("[MGMT] Starting interactive shell")
with app.app_context():
import IPython
......@@ -64,6 +66,20 @@ def cli_mgmt_arch_submissions():
IPython.embed()
@management_cli.command('load-script', help='DANGER ZONE - load script and '
'execute it with app context')
@click.argument('file')
def cli_mgmt_load_script(file):
print("[MGMT] loading script")
with app.app_context():
fp = Path(file).absolute()
if fp.exists():
content = fp.read_text('utf-8')
exec(content)
else:
print(f"[ERR] File not exists: {fp}")
@management_cli.command('delete-cancelled')
def cli_mgmt_arch_submissions():
print("[MGMT] Deleting cancelled submissions - NOW")
......
import shutil
from pathlib import Path
from portal import storage_wrapper
from portal.database import models
UPDATE_NAME = "transform_storage_dirs"
DEPENDS_ON = []
def change_dirs(entity, base_path: Path):
old_path = base_path / entity.id
new_path = base_path / entity.storage_dirname
if old_path.exists():
print(f"[MOVE] FROM {old_path} to {new_path}")
shutil.move(old_path, new_path)
def change_project_dirs():
print("[UPDATE] Processing projects")
tf_path = storage_wrapper.test_files.path
for project in models.Project.query.all():
change_dirs(project, tf_path)
def change_submission_dirs():
print("[UPDATE] Processing submissions")
source_path = storage_wrapper.submissions.path
results_path = storage_wrapper.results.path
for submission in models.Submission.query.all():
change_dirs(submission, source_path)
change_dirs(submission, results_path)
change_project_dirs()
change_submission_dirs()
......@@ -75,12 +75,12 @@ class SubmissionProcessor:
log.info(f"[ASYNC] Uploading submission: {self.submission.log_name} with {file_params}")
updated_entity: UploadedEntity = self.storage. \
submissions.create(dirname=self.submission.id, **file_params)
submissions.create(dirname=self.submission.storage_dirname, **file_params)
self.submission_store_ended(version=updated_entity.version)
def clone(self, target):
log.info(f"[ASYNC] Cloning submission: {self.submission.log_name} to {target.log_name}")
self.storage.submissions.clone(self.submission.id, target.id)
self.storage.submissions.clone(self.submission.storage_dirname, target.id)
self.submission_store_ended(version=self.submission.source_hash)
def send_to_worker(self):
......@@ -97,7 +97,7 @@ class SubmissionProcessor:
def upload_result(self, path, file_params):
log.info(f"[ASYNC] Uploading result for the submission "
f"{self.submission.log_name} with {file_params}")
self.storage.results.create(dirname=self.submission.id, **file_params)
self.storage.results.create(dirname=self.submission.storage_dirname, **file_params)
Path(path).unlink()
self.reset_task_id(SubmissionState.FINISHED)
......
......@@ -42,9 +42,7 @@ def archive_submission(submission_id: str):
@celery_app.task(name='upload-results-to-storage')
def upload_results_to_storage(new_submission_id: str, path: str):
path = str(path)
find_service = FindService()
new_submission = find_service.submission(new_submission_id)
new_submission = FindService().submission(new_submission_id)
log.info(f"[SUBMIT] Processing results - upload to the storage for "
f"{new_submission.log_name}: {path}")
SubmissionsService(new_submission).upload_results_to_storage(path)
......
......@@ -444,12 +444,13 @@ class Project(db.Model, EntityBase, NamedMixin):
def state(self) -> ProjectState:
return self.get_state_by_timestamp()
def get_state_by_timestamp(self, timestamp=time.current_time()) -> ProjectState:
def get_state_by_timestamp(self, timestamp=None) -> ProjectState:
"""Gets project state based on the timestamp
Args:
timestamp: Time for which the state should be calculated
Returns:
"""
"""
timestamp = timestamp or time.current_time()
if not (self.config.submissions_allowed_from or
self.config.submissions_allowed_to or
self.config.archive_from):
......@@ -862,7 +863,7 @@ class Submission(db.Model, EntityBase):
return f"{self.project.namespace}/{self.user.codename}/{self.created_at}"
@hybrid_property
def storage_dirname(self):
def storage_dirname(self) -> str:
created = time.simple_fmt(self.created_at)
return f"{self.user.username}_{created}_{self.course.codename}_{self.project.codename}"
......
import logging
from werkzeug.datastructures import FileStorage
from portal.async_celery import tasks
from portal.database import Project, Submission, SubmissionState, User
from portal.facade.general_facade import GeneralCRUDFacade
......@@ -81,10 +83,10 @@ class SubmissionsFacade(GeneralCRUDFacade):
def upload_results_to_storage(self, submission: Submission):
log.info(f"[UPLOAD] Uploading results to storage for "
f"{submission.log_name} by {self.client_name}")
file = self._request.files['file']
file: FileStorage = self._request.files['file']
submission_service: SubmissionsService = self._service(submission)
return tasks.upload_results_to_storage.delay(submission.id,
submission_service.get_upload_file_path(file))
path = submission_service.get_upload_file_path(file)
return tasks.upload_results_to_storage.delay(submission.id, path)
def copy_submission(self, source_submission: Submission, **params):
"""Copies a submission. Used at resubmitting
......
......@@ -6,6 +6,7 @@ import json
import logging
from pathlib import Path
from werkzeug.datastructures import FileStorage
from werkzeug.utils import secure_filename
from portal import storage_wrapper
......@@ -39,10 +40,11 @@ class SubmissionsService(GeneralService):
SubmissionsService(submission).archive()
@classmethod
def get_upload_file_path(cls, file) -> str:
upload_file_is_allowed(file)
path = upload_files_to_storage(file)
return str(path)
def get_upload_file_path(cls, file: FileStorage) -> str:
if upload_file_is_allowed(file):
path = upload_files_to_storage(file)
return str(path)
return None
def processor(self) -> SubmissionProcessor:
return SubmissionProcessor(self.submission)
......@@ -152,8 +154,7 @@ class SubmissionsService(GeneralService):
"""
self.processor().revoke_task()
def upload_results_to_storage(self, file):
path = self.get_upload_file_path(file)
def upload_results_to_storage(self, path):
processor = self.processor()
file_params = dict(source=dict(url=path, type='zip'))
processor.upload_result(path=path, file_params=file_params)
......@@ -224,7 +225,7 @@ def nonempty_intersection(provided: list, required: list):
return list(set(provided) & set(required))
def upload_files_to_storage(file):
def upload_files_to_storage(file: FileStorage):
filename = secure_filename(file.filename)
uploads = storage_wrapper.results.workspace_path / 'uploads'
if not uploads.exists():
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment