Verified Commit a6bd7ff8 authored by Peter Stanko's avatar Peter Stanko
Browse files

Storage service + refactor of getting test files

parent 103e565c
Pipeline #14168 failed with stage
in 60 minutes and 5 seconds
...@@ -193,6 +193,15 @@ class ProjectSubmissions(CustomResource): ...@@ -193,6 +193,15 @@ class ProjectSubmissions(CustomResource):
@projects_namespace.response(404, 'Course not found') @projects_namespace.response(404, 'Course not found')
@projects_namespace.response(404, 'Project not found') @projects_namespace.response(404, 'Project not found')
class ProjectTestFiles(CustomResource): class ProjectTestFiles(CustomResource):
@jwt_required
def get(self, cid: str, pid: str): def get(self, cid: str, pid: str):
# Gets test files (feature) course = self.find.course(cid)
raise NotImplementedError() self.permissions(course=course).require.view_course_full()
project = self.find.project(course, pid)
service = self.rest.storage(project=project)
storage_entity = service.get_test_files_entity_from_storage()
return service.send_file_or_zip(storage_entity)
...@@ -98,6 +98,9 @@ class FlaskRequestArgsHelper: ...@@ -98,6 +98,9 @@ class FlaskRequestArgsHelper:
return [] return []
return [self._rest.find.role(course, roleId) for roleId in roles] return [self._rest.find.role(course, roleId) for roleId in roles]
def state(self):
self._args.get('state')
class FlaskRequestHelper: class FlaskRequestHelper:
@property @property
......
...@@ -65,15 +65,6 @@ class SubmissionState(CustomResource): ...@@ -65,15 +65,6 @@ class SubmissionState(CustomResource):
return '', 204 return '', 204
@submissions_namespace.route('/<string:sid>/result')
@submissions_namespace.param('sid', 'Submission id')
@submissions_namespace.response(404, 'Submissions not found')
class SubmissionResult(CustomResource):
@jwt_required
def put(self, sid: str):
raise NotImplementedError()
@submissions_namespace.route('/<string:sid>/files') @submissions_namespace.route('/<string:sid>/files')
@submissions_namespace.param('sid', 'Submission id') @submissions_namespace.param('sid', 'Submission id')
@submissions_namespace.response(404, 'Submissions not found') @submissions_namespace.response(404, 'Submissions not found')
...@@ -92,7 +83,7 @@ class SubmissionSourcesTree(CustomResource): ...@@ -92,7 +83,7 @@ class SubmissionSourcesTree(CustomResource):
submission = self.find.submission(sid) submission = self.find.submission(sid)
course = submission.project.course course = submission.project.course
self.permissions(course=course).require.read_submission(submission) self.permissions(course=course).require.read_submission(submission)
service = self.rest.submissions(submission) service = self.rest.storage(submission)
return service.send_files_tree() return service.send_files_tree()
...@@ -105,7 +96,7 @@ class SubmissionSourceFiles(CustomResource): ...@@ -105,7 +96,7 @@ class SubmissionSourceFiles(CustomResource):
submission = self.find.submission(sid) submission = self.find.submission(sid)
course = submission.project.course course = submission.project.course
self.permissions(course=course).require.read_submission(submission) self.permissions(course=course).require.read_submission(submission)
service = self.rest.submissions(submission) service = self.rest.storage(submission)
return service.send_file_or_zip() return service.send_file_or_zip()
...@@ -118,7 +109,7 @@ class SubmissionTestFilesTree(CustomResource): ...@@ -118,7 +109,7 @@ class SubmissionTestFilesTree(CustomResource):
submission = self.find.submission(sid) submission = self.find.submission(sid)
course = submission.project.course course = submission.project.course
self.permissions(course=course).require.read_submission_group(submission) self.permissions(course=course).require.read_submission_group(submission)
service = self.rest.submissions(submission) service = self.rest.storage(project=submission.project)
storage_entity = service.get_test_files_entity_from_storage() storage_entity = service.get_test_files_entity_from_storage()
return service.send_files_tree(storage_entity) return service.send_files_tree(storage_entity)
...@@ -132,7 +123,7 @@ class SubmissionTestFiles(CustomResource): ...@@ -132,7 +123,7 @@ class SubmissionTestFiles(CustomResource):
submission = self.find.submission(sid) submission = self.find.submission(sid)
course = submission.project.course course = submission.project.course
self.permissions(course=course).require.read_submission_group(submission) self.permissions(course=course).require.read_submission_group(submission)
service = self.rest.submissions(submission) service = self.rest.storage(submission=submission)
storage_entity = service.get_test_files_entity_from_storage() storage_entity = service.get_test_files_entity_from_storage()
return service.send_file_or_zip(storage_entity) return service.send_file_or_zip(storage_entity)
...@@ -147,7 +138,7 @@ class SubmissionResultFilesTree(CustomResource): ...@@ -147,7 +138,7 @@ class SubmissionResultFilesTree(CustomResource):
course = submission.project.course course = submission.project.course
self.permissions(course=course).require.read_submission_group(submission) self.permissions(course=course).require.read_submission_group(submission)
storage_entity = storage.results.get(submission.id) storage_entity = storage.results.get(submission.id)
service = self.rest.submissions(submission) service = self.rest.storage(submission)
return service.send_files_tree(storage_entity) return service.send_files_tree(storage_entity)
...@@ -161,7 +152,7 @@ class SubmissionResultFiles(CustomResource): ...@@ -161,7 +152,7 @@ class SubmissionResultFiles(CustomResource):
course = submission.project.course course = submission.project.course
self.permissions(course=course).require.read_submission_group(submission) self.permissions(course=course).require.read_submission_group(submission)
storage_entity = storage.results.get(submission.id) storage_entity = storage.results.get(submission.id)
service = self.rest.submissions(submission) service = self.rest.storage(submission=submission)
return service.send_file_or_zip(storage_entity) return service.send_file_or_zip(storage_entity)
@jwt_required @jwt_required
......
...@@ -123,6 +123,9 @@ class PermissionServiceRequire: ...@@ -123,6 +123,9 @@ class PermissionServiceRequire:
def view_course(self): def view_course(self):
self._check.view_course_any() self._check.view_course_any()
def view_course_full(self):
self.permissions(['update_course', 'view_course_full'])
def belongs_to_group(self, group): def belongs_to_group(self, group):
checks = [ checks = [
self._check.view_course_full(), self._check.view_course_full(),
......
...@@ -14,11 +14,16 @@ from portal.service import errors, filters ...@@ -14,11 +14,16 @@ from portal.service import errors, filters
from portal.service.errors import ForbiddenError from portal.service.errors import ForbiddenError
from portal.service.general import GeneralService, get_new_name from portal.service.general import GeneralService, get_new_name
from portal.tools import time from portal.tools import time
from portal import storage
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
class ProjectService(GeneralService): class ProjectService(GeneralService):
@property
def storage(self):
return storage
@property @property
def _entity_klass(self): def _entity_klass(self):
return Project return Project
...@@ -152,4 +157,4 @@ class ProjectService(GeneralService): ...@@ -152,4 +157,4 @@ class ProjectService(GeneralService):
def update_project_test_files(self): def update_project_test_files(self):
""" Sends a request to Storage to update the project's test_files to the newest version. """ Sends a request to Storage to update the project's test_files to the newest version.
""" """
tasks.update_project_test_files.delay(self.project.course.id, self.project.id) tasks.update_project_test_files.delay(self.project.course.id, self.project.id)
\ No newline at end of file
...@@ -7,6 +7,7 @@ from portal.service.projects import ProjectService ...@@ -7,6 +7,7 @@ from portal.service.projects import ProjectService
from portal.service.reviews import ReviewService from portal.service.reviews import ReviewService
from portal.service.roles import RoleService from portal.service.roles import RoleService
from portal.service.secrets import SecretsService from portal.service.secrets import SecretsService
from portal.service.storage import StorageService
from portal.service.submissions import SubmissionsService from portal.service.submissions import SubmissionsService
from portal.service.users import UserService from portal.service.users import UserService
from portal.service.workers import WorkerService from portal.service.workers import WorkerService
...@@ -64,3 +65,7 @@ class RestService: ...@@ -64,3 +65,7 @@ class RestService:
@property @property
def reviews(self) -> ReviewService: def reviews(self) -> ReviewService:
return ReviewService(self) return ReviewService(self)
@property
def storage(self) -> StorageService:
return StorageService(self)
import time
from typing import Optional
import flask
from storage import entities
from portal import logger, storage
from portal.database import Project, Submission
from portal.service import errors
from portal.service.general import GeneralService
log = logger.get_logger(__name__)
class StorageService(GeneralService):
def _set_data(self, entity, data=None):
return None
@property
def storage(self):
return storage
def __call__(self, submission=None, project=None):
self._project = project
self._submission = submission
return self
@property
def storage_entity(self) -> entities.Entity:
if self._submission is not None:
return self.storage.submissions.get(self._submission.id)
if self._project is not None:
return self.storage.test_files.get(self._project.id)
@property
def submission(self) -> Submission:
return self._submission
@property
def project(self) -> Optional[Project]:
if self._project is not None:
return self._project
if self.submission is not None:
return self.submission.project
return None
def __init__(self, rest_service):
super().__init__(rest_service)
self._submission = None
self._project = None
def send_zip(self, storage_entity: entities.Submission):
storage_entity = storage_entity or self.storage_entity
path = storage_entity.zip_path
klass = storage_entity.__class__.__name__
if not path.exists():
raise errors.PortalAPIError(400, f"Requested path does not exist: {path}")
log.debug(f"[SEND] Sending zip file {klass} "
f"({self.submission.id}): {path}")
return flask.send_file(str(path), attachment_filename=path.name)
def send_selected_file(self, storage_entity: entities.Submission, path_query: str):
storage_entity = storage_entity or self.storage_entity
klass = storage_entity.__class__.__name__
log_name = self.submission.log_name if self.submission else self.project.log_name
log.debug(f"[SEND] Sending file {klass} {log_name}: {path_query}")
path = storage_entity.get(path_query)
return flask.send_file(str(path), attachment_filename=path.name)
def send_file_or_zip(self, storage_entity=None):
storage_entity = storage_entity or self.storage_entity
path_query = self.request.args.get('path')
if path_query is None:
return self.send_zip(storage_entity)
return self.send_selected_file(storage_entity, path_query)
def send_files_tree(self, storage_entity=None):
storage_entity = storage_entity or self.storage_entity
tree = storage_entity.tree()
log.debug(f"[TREE] Tree for the storage entity {storage_entity.entity_id}: {tree}")
return tree
def get_test_files_entity_from_storage(self, project=None):
project = project or self.project
if not project.config.test_files_commit_hash:
log.warning(f"Test files are not present "
f"for the project {project.log_name}")
self._rest_service.projects(project).update_project_test_files()
self.__wait_for_test_files()
return storage.test_files.get(project.id)
def __wait_for_test_files(self):
project = self.project
wait_interval = 60
log.debug("[WAIT] Waiting for test files to be present")
while True:
self.refresh(project)
wait_interval -= 1
if wait_interval <= 0:
raise TimeoutError("Waiting for the test files timed out")
log.debug(f"[WAIT] Project Files Hash: {project.config.test_files_commit_hash}")
if project.config.test_files_commit_hash is not None:
break
time.sleep(1)
...@@ -5,16 +5,14 @@ Submissions service ...@@ -5,16 +5,14 @@ Submissions service
import logging import logging
import time import time
from pathlib import Path from pathlib import Path
from typing import Union, List from typing import List, Union
import flask
from celery.result import AsyncResult from celery.result import AsyncResult
from storage import entities
from werkzeug.utils import secure_filename from werkzeug.utils import secure_filename
from portal import storage from portal import storage
from portal.async_celery import submission_processor, tasks from portal.async_celery import submission_processor, tasks
from portal.database.models import Project, Submission, SubmissionState, User, Worker, Role, Group from portal.database.models import Group, Project, Role, Submission, SubmissionState, User, Worker
from portal.rest.rest_helpers import FlaskRequestHelper from portal.rest.rest_helpers import FlaskRequestHelper
from portal.service import errors from portal.service import errors
from portal.service.general import GeneralService from portal.service.general import GeneralService
...@@ -93,10 +91,6 @@ class SubmissionsService(GeneralService): ...@@ -93,10 +91,6 @@ class SubmissionsService(GeneralService):
def storage(self): def storage(self):
return storage return storage
@property
def storage_submission(self):
return storage.submissions.get(self.submission.id)
def process_new_submission(self) -> AsyncResult: def process_new_submission(self) -> AsyncResult:
project = self.submission.project project = self.submission.project
self.submission.parameters['file_params'] = project.config.file_whitelist self.submission.parameters['file_params'] = project.config.file_whitelist
...@@ -161,36 +155,6 @@ class SubmissionsService(GeneralService): ...@@ -161,36 +155,6 @@ class SubmissionsService(GeneralService):
processor = submission_processor.SubmissionProcessor(self.submission) processor = submission_processor.SubmissionProcessor(self.submission)
processor.revoke_task() processor.revoke_task()
def send_zip(self, storage_submission: entities.Submission):
storage_submission = storage_submission or self.storage_submission
path = storage_submission.zip_path
klass = self.storage_submission.__class__.__name__
if not path.exists():
raise errors.PortalAPIError(400, f"Requested path does not exist: {path}")
log.debug(f"[SEND] Sending zip file {klass} "
f"({self.submission.id}): {path}")
return flask.send_file(str(path), attachment_filename=path.name)
def send_selected_file(self, storage_entity: entities.Submission, path_query: str):
storage_entity = storage_entity or self.storage_submission
klass = self.storage_submission.__class__.__name__
log.debug(f"[SEND] Sending file {klass} {self.submission.log_name}: {path_query}")
path = storage_entity.get(path_query)
return flask.send_file(str(path), attachment_filename=path.name)
def send_file_or_zip(self, storage_entity=None):
storage_entity = storage_entity or self.storage_submission
path_query = self.request.args.get('path')
if path_query is None:
return self.send_zip(storage_entity)
return self.send_selected_file(storage_entity, path_query)
def send_files_tree(self, storage_entity=None):
storage_entity = storage_entity or self.storage_submission
tree = storage_entity.tree()
log.debug(f"[TREE] Tree for the storage entity {storage_entity.entity_id}: {tree} ")
return tree
def upload_results_to_storage(self): def upload_results_to_storage(self):
path = self.get_upload_file_path() path = self.get_upload_file_path()
task = tasks.upload_results_to_storage.delay(self.submission.id, path=str(path)) task = tasks.upload_results_to_storage.delay(self.submission.id, path=str(path))
...@@ -202,29 +166,6 @@ class SubmissionsService(GeneralService): ...@@ -202,29 +166,6 @@ class SubmissionsService(GeneralService):
path = upload_files_to_storage(file) path = upload_files_to_storage(file)
return path return path
def get_test_files_entity_from_storage(self):
project = self.project
if not project.config.test_files_commit_hash:
log.warning(f"Test files are not present "
f"for the project {project.log_name}")
self._rest_service.projects(project).update_project_test_files()
self.__wait_for_test_files()
return storage.test_files.get(project.id)
def __wait_for_test_files(self):
project = self.project
wait_interval = 60
log.debug("[WAIT] Waiting for test files to be present")
while True:
self.refresh(project)
wait_interval -= 1
if wait_interval <= 0:
raise TimeoutError("Waiting for the test files timed out")
log.debug(f"[WAIT] Project Files Hash: {project.config.test_files_commit_hash}")
if project.config.test_files_commit_hash is not None:
break
time.sleep(1)
def filter_user_avail_submissions(self, query, roles: List[Role], groups: List[Group]): def filter_user_avail_submissions(self, query, roles: List[Role], groups: List[Group]):
submissions = query.all() submissions = query.all()
return [submission for submission in submissions return [submission for submission in submissions
...@@ -242,10 +183,14 @@ class SubmissionsService(GeneralService): ...@@ -242,10 +183,14 @@ class SubmissionsService(GeneralService):
projects = request_helper.args.projects() projects = request_helper.args.projects()
roles = request_helper.args.roles() roles = request_helper.args.roles()
groups = request_helper.args.groups() groups = request_helper.args.groups()
state = request_helper.args.state()
if user: if user:
query = query.filter(Submission.user == user) query = query.filter(Submission.user == user)
if state:
query = query.filter(Submission.state == state)
if course: if course:
query = query.filter(Submission.course == course) query = query.filter(Submission.course == course)
......
...@@ -74,6 +74,15 @@ def test_submission_test_files_are_available(client, mocked_submission): ...@@ -74,6 +74,15 @@ def test_submission_test_files_are_available(client, mocked_submission):
assert response.data.decode('utf-8') == expected('test_main.c') assert response.data.decode('utf-8') == expected('test_main.c')
def test_project_test_files_are_available(client, mocked_submission):
s = mocked_submission
url = f'/courses/{s.course.id}/projects/{s.project.id}/files?path=test_main.c'
response = utils.make_request(client, url)
assert response.status_code == 200
assert response.data
assert response.data.decode('utf-8') == expected('test_main.c')
def test_submission_sources_tree(client, mocked_submission): def test_submission_sources_tree(client, mocked_submission):
s = mocked_submission s = mocked_submission
response = utils.make_request(client, f'/submissions/{s.id}/files/sources/tree') response = utils.make_request(client, f'/submissions/{s.id}/files/sources/tree')
......
...@@ -47,7 +47,7 @@ class EntitiesMocker: ...@@ -47,7 +47,7 @@ class EntitiesMocker:
return self.data.create_submission(project=project, user=user) return self.data.create_submission(project=project, user=user)
def create_submission_storage(self, submission): def create_submission_storage(self, submission):
storage = self.rest.submissions.storage storage = self.rest.storage.storage
def __create_content(location: Entity, *files): def __create_content(location: Entity, *files):
path = location.path path = location.path
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment