Verified Commit 500de1b3 authored by Peter Stanko's avatar Peter Stanko
Browse files

Major storage and async tasks refactor

parent 3730157e
......@@ -166,17 +166,21 @@ class DataFactory:
log.debug(f"[CREATE] Project config: {project.log_name}: {config}")
project.set_config(**config)
self.session.flush()
test_path = storage_wrapper.test_files.path / project.id
test_path = storage_wrapper.test_files.path / project.storage_dirname
if test_path.exists():
shutil.rmtree(str(test_path))
shutil.copytree(str(TESTS_DIR), str(test_path))
return project
def create_submission(self, project, user, **kwargs):
submission = self.__create_entity(Submission, project=project, user=user, **kwargs)
self.session.flush()
subm_path = storage_wrapper.submissions.path / submission.id
shutil.copytree(str(SOURCES_DIR), str(subm_path))
res_path = storage_wrapper.results.path / submission.id
shutil.copytree(str(RESULTS_DIR), str(res_path))
subm_path = storage_wrapper.submissions.path / submission.storage_dirname
if not subm_path.exists():
shutil.copytree(str(SOURCES_DIR), str(subm_path))
res_path = storage_wrapper.results.path / submission.storage_dirname
if not res_path.exists():
shutil.copytree(str(RESULTS_DIR), str(res_path))
return submission
def create_review_item(self, review, author, file="main.c", line=1,
......
......@@ -13,11 +13,10 @@ log = logger.get_logger(__name__)
class SubmissionProcessor:
def __init__(self, submission: Submission, params: dict = None):
def __init__(self, submission: Submission):
self._submission = submission
self._params = params
from portal.service.services_collection import ServicesCollection
self._rest = ServicesCollection()
self._services = ServicesCollection()
@property
def submission(self) -> Submission:
......@@ -59,7 +58,7 @@ class SubmissionProcessor:
def dispatch_submission_processing(self):
delay = self.get_delay_for_submission()
args = (self.submission.id, self.params)
args = (self.submission.id,)
from .tasks import start_processing_submission
self.submission.scheduled_for = delay
self._save_submission()
......@@ -76,7 +75,7 @@ class SubmissionProcessor:
log.info(f"[ASYNC] Uploading submission: {self.submission.log_name} with {file_params}")
updated_entity: UploadedEntity = self.storage. \
submissions.create(entity_id=self.submission.id, **file_params)
submissions.create(dirname=self.submission.id, **file_params)
self.submission_store_ended(version=updated_entity.version)
def clone(self, target):
......@@ -90,11 +89,15 @@ class SubmissionProcessor:
worker = self.schedule_submission_to_worker()
if worker:
self.execute_submission(worker)
else:
log.warning(f"[EXEC] Worker not available to process submission: "
f"{self.submission.log_name}")
self.abort_submission("No Worker available")
def upload_result(self, path, file_params):
log.info(f"[ASYNC] Uploading result for the submission "
f"{self.submission.log_name} with {file_params}")
self.storage.results.create(entity_id=self.submission.id, **file_params)
self.storage.results.create(dirname=self.submission.id, **file_params)
Path(path).unlink()
self.reset_task_id(SubmissionState.FINISHED)
......@@ -118,7 +121,7 @@ class SubmissionProcessor:
def _get_avail_workers(self):
course = self.submission.course
workers = self._rest.workers.find_all()
workers = self._services.workers.find_all()
return [worker for worker in workers
if worker.state == WorkerState.READY and course in worker.courses]
......@@ -139,7 +142,7 @@ class SubmissionProcessor:
return worker
def execute_submission(self, worker: Worker):
worker_client = self._rest.workers(worker).worker_client
worker_client = self._services.workers(worker).worker_client
self.submission.change_state(SubmissionState.IN_PROGRESS)
self._save_submission()
worker_client.execute_submission(self.submission)
......@@ -148,7 +151,7 @@ class SubmissionProcessor:
log.warning(f"[PROC] Worker is no available for submission: {self.submission.log_name}")
def process_result(self):
storage_entity = self.storage.results.get(self.submission.id)
storage_entity = self.storage.results.get(self.submission.storage_dirname)
# @mdujava - here put submission processing
return self._submission_result_processing(storage_entity)
......@@ -156,7 +159,7 @@ class SubmissionProcessor:
suite_stats = storage_entity.get('suite-stats.json')
if not suite_stats.exists():
log.error(f"[PROC] Suite stats for the {self.submission.log_name} have not been found.")
raise errors.SuiteStatsNotExists(self.submission.id)
raise errors.SuiteStatsNotExists(self.submission.storage_dirname)
stats = json.loads(suite_stats.read_text('utf-8'))
return self._parse_stats(stats)
......@@ -171,7 +174,8 @@ class SubmissionProcessor:
def abort_submission(self, message: str = 'Unknown error!'):
self.submission.note['error'] = message
self._save_submission()
self.reset_task_id(SubmissionState.ABORTED)
def _save_submission(self):
self._rest.submissions.write_entity(self.submission)
self._services.submissions.write_entity(self.submission)
from celery.utils.log import get_task_logger
from portal.storage import UploadedEntity
from portal import storage_wrapper
from portal.async_celery import celery_app, submission_processor
from portal.service.courses import CourseService
from portal.service.find import FindService
from portal.service.general import GeneralService
from portal.service.is_api_service import IsApiService
from portal.service.projects import ProjectService
from portal.service.submissions import SubmissionsService
log = get_task_logger(__name__)
"""
= Submissions related
"""
@celery_app.task(name='upload-submission-to-storage')
def process_submission(new_submission_id: str):
find_service = FindService()
new_submission = find_service.submission(new_submission_id)
project = new_submission.project
course = project.course
log.info(f"[SUBMIT] Processing submission: {new_submission.log_name}")
if not project.config.test_files_commit_hash:
log.warning(f"Project test files not found: {project.log_name}")
update_project_test_files(course_id=course.id, project_id=project.id)
update_project_test_files(course_id=project.course.id, project_id=project.id)
new_submission = find_service.submission(new_submission_id)
processor = submission_processor.SubmissionProcessor(new_submission)
processor.process_submission()
@celery_app.task(name='delete-submission')
def delete_submission(submission_id: str):
submission = FindService().submission(submission_id)
SubmissionsService(submission).delete()
@celery_app.task(name='archive-submission')
def archive_submission(submission_id: str):
submission = FindService().submission(submission_id)
SubmissionsService(submission).archive()
@celery_app.task(name='upload-results-to-storage')
def upload_results_to_storage(new_submission_id: str, path: str):
path = str(path)
......@@ -33,11 +47,7 @@ def upload_results_to_storage(new_submission_id: str, path: str):
new_submission = find_service.submission(new_submission_id)
log.info(f"[SUBMIT] Processing results - upload to the storage for "
f"{new_submission.log_name}: {path}")
processor = submission_processor.SubmissionProcessor(new_submission)
file_params = dict(source=dict(url=path, type='zip'))
processor.upload_result(path=path, file_params=file_params)
submission = processor.process_result()
SubmissionsService().write_entity(submission)
SubmissionsService(new_submission).upload_results_to_storage(path)
@celery_app.task(name='clone-submission-files')
......@@ -50,13 +60,18 @@ def clone_submission_files(source_id: str, target_id: str):
@celery_app.task(name='start-processing-submission')
def start_processing_submission(submission_id: str, submission_params):
def start_processing_submission(submission_id: str):
submission = FindService().submission(submission_id)
log.info(f"[SUBMIT] Processing submission - send to worker: {submission.log_name}")
processor = submission_processor.SubmissionProcessor(submission, submission_params)
processor = submission_processor.SubmissionProcessor(submission)
processor.send_to_worker()
"""
= Project related
"""
@celery_app.task(name='update-project-test-files')
def update_project_test_files(course_id: str, project_id: str):
find_service = FindService()
......@@ -71,11 +86,47 @@ def update_project_test_files(course_id: str, project_id: str):
}
}
try:
_update_test_files(project, params)
ProjectService(project).update_project_test_files(params)
except Exception as ex:
log.error(f"[ASYNC] Cannot update source files {project.log_name}: {ex}")
@celery_app.task(name='delete-project')
def delete_project(course_id, project_id: str):
course = FindService().course(course_id)
submission = FindService().project(course, project_id)
ProjectService(submission).delete()
@celery_app.task(name='archive-project')
def archive_project(course_id, project_id: str):
course = FindService().course(course_id)
project = FindService().project(course, project_id)
ProjectService(project).archive()
"""
= COURSE RELATED
"""
@celery_app.task(name='delete-course')
def delete_course(course_id):
course = FindService().course(course_id)
CourseService(course).delete()
@celery_app.task(name='archive-course')
def archive_course(course_id):
course = FindService().course(course_id)
CourseService(course).archive()
"""
= IS MUNI RELATED
"""
@celery_app.task(name='is-sync-course-seminaries')
def is_sync_seminaries(course_id):
find_service = FindService()
......@@ -90,12 +141,3 @@ def is_import_users_for_course(course_id: str, role_name: str, users_type: str):
course = find_service.course(course_id)
log.info(f"[ASYNC] Import course users for {course.log_name}: {users_type} -> {role_name}")
IsApiService(course).import_users(role_name=role_name, users_type=users_type)
def _update_test_files(project, params):
updated_entity: UploadedEntity = storage_wrapper.test_files.update(entity_id=project.id, **params)
version = updated_entity.version
project.config.test_files_commit_hash = version
log.debug(f"Updated project config {project.log_name}: {project.config}")
GeneralService().write_entity(project.config)
......@@ -436,6 +436,10 @@ class Project(db.Model, EntityBase, NamedMixin):
def namespace(self) -> str:
return f"{self.course.codename}/{self.codename}"
@hybrid_property
def storage_dirname(self):
return f"{self.course.codename}_{self.codename}"
@hybrid_property
def state(self) -> ProjectState:
return self.get_state_by_timestamp()
......@@ -857,6 +861,11 @@ class Submission(db.Model, EntityBase):
def namespace(self) -> str:
return f"{self.project.namespace}/{self.user.codename}/{self.created_at}"
@hybrid_property
def storage_dirname(self):
created = time.simple_fmt(self.created_at)
return f"{self.user.username}_{created}_{self.course.codename}_{self.project.codename}"
def change_state(self, new_state):
# open to extension (state transition validation, ...)
if new_state in Submission.ALLOWED_TRANSITIONS.keys():
......
from portal import facade
class FacadesCollection:
_collection = None
@classmethod
def get(cls) -> 'FacadesCollection':
if cls._collection is None:
cls._collection = cls()
return cls._collection
def __init__(self):
self._reviews = facade.ReviewsFacade()
self._users = facade.UsersFacade()
self._courses = facade.CoursesFacade()
self._roles = facade.RolesFacade()
self._groups = facade.GroupsFacade()
self._secrets = facade.SecretsFacade()
self._workers = facade.WorkersFacade()
self._submissions = facade.SubmissionsFacade()
self._projects = facade.ProjectsFacade()
self._gitlab = facade.GitlabFacade()
@property
def users(self) -> facade.UsersFacade:
return self._users
@property
def courses(self) -> facade.CoursesFacade:
return self._courses
@property
def groups(self) -> facade.GroupsFacade:
return self._groups
@property
def roles(self) -> facade.RolesFacade:
return self._roles
@property
def secrets(self) -> facade.SecretsFacade:
return self._secrets
@property
def submissions(self) -> facade.SubmissionsFacade:
return self._submissions
@property
def projects(self) -> facade.ProjectsFacade:
return self._projects
@property
def workers(self) -> facade.WorkersFacade:
return self._workers
@property
def reviews(self) -> facade.ReviewsFacade:
return self._reviews
@property
def gitlab(self) -> facade.GitlabFacade:
return self._gitlab
\ No newline at end of file
......@@ -3,8 +3,8 @@ from typing import List
from portal.async_celery import tasks
from portal.database import Course
from portal.database.models import Client
from portal.database.enums import ClientType
from portal.database.models import Client
from portal.facade.general_facade import GeneralCRUDFacade
from portal.service.courses import CourseService
......@@ -76,3 +76,14 @@ class CoursesFacade(GeneralCRUDFacade):
task = tasks.is_sync_seminaries.delay(course.id)
log.debug(f"[TASK] Created debug task: {task}")
return task
def delete(self, course: Course):
for project in course.projects:
self._facades.projects.delete(project)
super(CoursesFacade, self).delete(course)
def archive(self, course: Course):
for project in course.projects:
self._facades.projects.archive(project)
self._service(course).archive()
......@@ -27,6 +27,11 @@ class GeneralFacade:
self._services = ServicesCollection()
self._urls = UrlService()
@property
def _facades(self):
from portal.facade.collection import FacadesCollection
return FacadesCollection.get()
@property
def urls(self):
return self._urls
......
import logging
from portal.async_celery import tasks
from portal.database import Project
from portal.database.models import Client, Course
from portal.facade.general_facade import GeneralCRUDFacade
......@@ -71,9 +72,9 @@ class ProjectsFacade(GeneralCRUDFacade):
project: project instance
"""
# TODO invoke the async task
result = self._service(project).update_project_test_files()
return result
# TODO: Wrap tasks call to service
log.info(f"[UPDATE] Project files for project {project}")
return tasks.update_project_test_files.delay(project.course.id, project.id)
def find_project_submissions(self, project: Project, user_id: str):
"""Finds all project submissions. If a user id is specified, returns only submissions created
......@@ -142,3 +143,12 @@ class ProjectsFacade(GeneralCRUDFacade):
_check_gl_repo_visibility(gl_project, project)
return True
def delete(self, project: Project):
for submission in project.submissions:
self._facades.submissions.delete(submission)
super(ProjectsFacade, self).delete(project)
def archive(self, project: Project):
for submission in project.submissions:
self._facades.submissions.archive(submission)
self._service(project).archive()
import logging
from portal.async_celery import tasks
from portal.database import Project, Submission, SubmissionState, User
from portal.facade.general_facade import GeneralCRUDFacade
from portal.logger import SUBMIT
......@@ -13,7 +14,7 @@ class SubmissionsFacade(GeneralCRUDFacade):
def __init__(self):
super().__init__(SubmissionsService, 'Submission')
def create(self, project, user, **data):
def create(self, project, user, evaluate=True, **data):
from portal.tools import request_helpers
SUBMIT.info(f'[SUBMIT] Submission for project: "{project.log_name}", '
f'user: "{user.log_name}", created: "{self.client_name}", '
......@@ -21,8 +22,11 @@ class SubmissionsFacade(GeneralCRUDFacade):
f'({request_helpers.get_ip()})')
user = user or self.client
allow_override = self.permissions(course=project.course).check.create_submission_other()
created_submission = super(SubmissionsFacade, self) \
created_submission: Submission = super(SubmissionsFacade, self) \
.create(user=user, project=project, allow_override=allow_override, **data)
if evaluate:
log.info(f"[SUBMIT] Evaluation of submission: {created_submission.log_name}")
tasks.process_submission.delay(created_submission.id)
return created_submission
def get_source_files(self, submission: Submission):
......@@ -32,7 +36,7 @@ class SubmissionsFacade(GeneralCRUDFacade):
def get_result_files(self, submission: Submission):
service = self.services.storage(submission)
storage_entity = self.storage.results.get(submission.id)
storage_entity = self.storage.results.get(submission.storage_dirname)
path = self._request.args.get('path')
return service.send_file_or_zip(path=path, storage_entity=storage_entity)
......@@ -43,7 +47,7 @@ class SubmissionsFacade(GeneralCRUDFacade):
return service.send_file_or_zip(storage_entity=storage_entity, path=path)
def result_files_tree(self, submission: Submission):
storage_entity = self.storage.results.get(submission.id)
storage_entity = self.storage.results.get(submission.storage_dirname)
service = self.services.storage(submission)
return service.send_files_tree(storage_entity)
......@@ -78,7 +82,9 @@ class SubmissionsFacade(GeneralCRUDFacade):
log.info(f"[UPLOAD] Uploading results to storage for "
f"{submission.log_name} by {self.client_name}")
file = self._request.files['file']
return self._service(submission).upload_results_to_storage(file)
submission_service: SubmissionsService = self._service(submission)
return tasks.upload_results_to_storage.delay(submission.id,
submission_service.get_upload_file_path(file))
def copy_submission(self, source_submission: Submission, **params):
"""Copies a submission. Used at resubmitting
......@@ -90,19 +96,23 @@ class SubmissionsFacade(GeneralCRUDFacade):
"""
log.info(f"[COPY] Submission {source_submission.log_name} to "
f"same project {source_submission.project.log_name} by {self.client_name}")
return self._service(source_submission).copy_submission(**params)
submission = self._service(source_submission).copy_submission(**params)
tasks.clone_submission_files.delay(source_id=source_submission.id,
target_id=submission.id)
return submission
def resend_submission(self, submission: Submission):
""" Resends the submission to the worker
"""
# TODO: ASYNC TASK
log.info(f"[SUBMIT] Resending submission task for {submission.log_name}")
return self._service(submission).resend_submission()
return tasks.start_processing_submission.delay(submission.id)
def cancel_submission(self, submission: Submission):
log.info(f"[CANCEL] Cancelling the submission {submission.log_name} "
f"by {self.client_name}")
return self._service(submission).cancel_submission()
self._service(submission).cancel_submission()
tasks.delete_submission.delay(submission.id)
return
def find_all(self, *args, **kwargs):
role_ids = self._request.args.getlist('roles')
......
import flask
from flask_restplus import Resource
from portal import facade
from portal.database.models import Client
from portal.facade.collection import FacadesCollection
from portal.service import errors
from portal.service.auth import AuthService
from portal.service.find import FindService
......@@ -10,68 +10,6 @@ from portal.service.permissions import PermissionsService
from portal.service.services_collection import ServicesCollection
class FacadesCollection:
_collection = None
@classmethod
def get(cls) -> 'FacadesCollection':
if cls._collection is None:
cls._collection = cls()
return cls._collection
def __init__(self):
self._reviews = facade.ReviewsFacade()
self._users = facade.UsersFacade()
self._courses = facade.CoursesFacade()
self._roles = facade.RolesFacade()
self._groups = facade.GroupsFacade()
self._secrets = facade.SecretsFacade()
self._workers = facade.WorkersFacade()
self._submissions = facade.SubmissionsFacade()
self._projects = facade.ProjectsFacade()
self._gitlab = facade.GitlabFacade()
@property
def users(self) -> facade.UsersFacade:
return self._users
@property
def courses(self) -> facade.CoursesFacade:
return self._courses
@property
def groups(self) -> facade.GroupsFacade:
return self._groups
@property
def roles(self) -> facade.RolesFacade:
return self._roles
@property
def secrets(self) -> facade.SecretsFacade:
return self._secrets
@property
def submissions(self) -> facade.SubmissionsFacade:
return self._submissions
@property
def projects(self) -> facade.ProjectsFacade:
return self._projects
@property
def workers(self) -> facade.WorkersFacade:
return self._workers
@property
def reviews(self) -> facade.ReviewsFacade:
return self._reviews
@property
def gitlab(self) -> facade.GitlabFacade:
return self._gitlab
class CustomResource(Resource):
@property
def services(self) -> ServicesCollection:
......
......@@ -86,6 +86,6 @@ def handle_portal_api_error(ex: PortalAPIError):
@rest_api.errorhandler(Exception)
def handle_default_exception(ex: Exception):
log.critical(f"[ERROR] Fatal error: {ex}")
if flask.current_app.config.get('TESTING') is True: