Storage service + refactor of getting test files

portal/rest/projects.py

@@ -193,6 +193,15 @@ class ProjectSubmissions(CustomResource):
 @projects_namespace.response(404, 'Course not found')
 @projects_namespace.response(404, 'Project not found')
 class ProjectTestFiles(CustomResource):
+    @jwt_required
     def get(self, cid: str, pid: str):
-        # Gets test files (feature)
-        raise NotImplementedError()
+        course = self.find.course(cid)
+        self.permissions(course=course).require.view_course_full()
+        project = self.find.project(course, pid)
+        service = self.rest.storage(project=project)
+        storage_entity = service.get_test_files_entity_from_storage()
+        return service.send_file_or_zip(storage_entity)
+
+
+
+

portal/rest/rest_helpers.py

@@ -98,6 +98,9 @@ class FlaskRequestArgsHelper:
             return []
         return [self._rest.find.role(course, roleId) for roleId in roles]
 
+    def state(self):
+        self._args.get('state')
+
 
 class FlaskRequestHelper:
     @property

portal/rest/submissions.py

@@ -65,15 +65,6 @@ class SubmissionState(CustomResource):
         return '', 204
 
 
-@submissions_namespace.route('/<string:sid>/result')
-@submissions_namespace.param('sid', 'Submission id')
-@submissions_namespace.response(404, 'Submissions not found')
-class SubmissionResult(CustomResource):
-    @jwt_required
-    def put(self, sid: str):
-        raise NotImplementedError()
-
-
 @submissions_namespace.route('/<string:sid>/files')
 @submissions_namespace.param('sid', 'Submission id')
 @submissions_namespace.response(404, 'Submissions not found')
@@ -92,7 +83,7 @@ class SubmissionSourcesTree(CustomResource):
         submission = self.find.submission(sid)
         course = submission.project.course
         self.permissions(course=course).require.read_submission(submission)
-        service = self.rest.submissions(submission)
+        service = self.rest.storage(submission)
         return service.send_files_tree()
 
 
@@ -105,7 +96,7 @@ class SubmissionSourceFiles(CustomResource):
         submission = self.find.submission(sid)
         course = submission.project.course
         self.permissions(course=course).require.read_submission(submission)
-        service = self.rest.submissions(submission)
+        service = self.rest.storage(submission)
         return service.send_file_or_zip()
 
 
@@ -118,7 +109,7 @@ class SubmissionTestFilesTree(CustomResource):
         submission = self.find.submission(sid)
         course = submission.project.course
         self.permissions(course=course).require.read_submission_group(submission)
-        service = self.rest.submissions(submission)
+        service = self.rest.storage(project=submission.project)
         storage_entity = service.get_test_files_entity_from_storage()
         return service.send_files_tree(storage_entity)
 
@@ -132,7 +123,7 @@ class SubmissionTestFiles(CustomResource):
         submission = self.find.submission(sid)
         course = submission.project.course
         self.permissions(course=course).require.read_submission_group(submission)
-        service = self.rest.submissions(submission)
+        service = self.rest.storage(submission=submission)
         storage_entity = service.get_test_files_entity_from_storage()
         return service.send_file_or_zip(storage_entity)
 
@@ -147,7 +138,7 @@ class SubmissionResultFilesTree(CustomResource):
         course = submission.project.course
         self.permissions(course=course).require.read_submission_group(submission)
         storage_entity = storage.results.get(submission.id)
-        service = self.rest.submissions(submission)
+        service = self.rest.storage(submission)
         return service.send_files_tree(storage_entity)
 
 
@@ -161,7 +152,7 @@ class SubmissionResultFiles(CustomResource):
         course = submission.project.course
         self.permissions(course=course).require.read_submission_group(submission)
         storage_entity = storage.results.get(submission.id)
-        service = self.rest.submissions(submission)
+        service = self.rest.storage(submission=submission)
         return service.send_file_or_zip(storage_entity)
 
     @jwt_required

portal/service/permissions.py

@@ -123,6 +123,9 @@ class PermissionServiceRequire:
     def view_course(self):
         self._check.view_course_any()
 
+    def view_course_full(self):
+        self.permissions(['update_course', 'view_course_full'])
+
     def belongs_to_group(self, group):
         checks = [
             self._check.view_course_full(),

portal/service/projects.py

@@ -14,11 +14,16 @@ from portal.service import errors, filters
 from portal.service.errors import ForbiddenError
 from portal.service.general import GeneralService, get_new_name
 from portal.tools import time
+from portal import storage
 
 log = logging.getLogger(__name__)
 
 
 class ProjectService(GeneralService):
+    @property
+    def storage(self):
+        return storage
+
     @property
     def _entity_klass(self):
         return Project

portal/service/rest.py

@@ -7,6 +7,7 @@ from portal.service.projects import ProjectService
 from portal.service.reviews import ReviewService
 from portal.service.roles import RoleService
 from portal.service.secrets import SecretsService
+from portal.service.storage import StorageService
 from portal.service.submissions import SubmissionsService
 from portal.service.users import UserService
 from portal.service.workers import WorkerService
@@ -64,3 +65,7 @@ class RestService:
     @property
     def reviews(self) -> ReviewService:
         return ReviewService(self)
+
+    @property
+    def storage(self) -> StorageService:
+        return StorageService(self)

portal/service/storage.py

+import time
+from typing import Optional
+
+import flask
+from storage import entities
+
+from portal import logger, storage
+from portal.database import Project, Submission
+from portal.service import errors
+from portal.service.general import GeneralService
+
+log = logger.get_logger(__name__)
+
+
+class StorageService(GeneralService):
+    def _set_data(self, entity, data=None):
+        return None
+
+    @property
+    def storage(self):
+        return storage
+
+    def __call__(self, submission=None, project=None):
+        self._project = project
+        self._submission = submission
+        return self
+
+    @property
+    def storage_entity(self) -> entities.Entity:
+        if self._submission is not None:
+            return self.storage.submissions.get(self._submission.id)
+        if self._project is not None:
+            return self.storage.test_files.get(self._project.id)
+
+    @property
+    def submission(self) -> Submission:
+        return self._submission
+
+    @property
+    def project(self) -> Optional[Project]:
+        if self._project is not None:
+            return self._project
+        if self.submission is not None:
+            return self.submission.project
+        return None
+
+    def __init__(self, rest_service):
+        super().__init__(rest_service)
+        self._submission = None
+        self._project = None
+
+    def send_zip(self, storage_entity: entities.Submission):
+        storage_entity = storage_entity or self.storage_entity
+        path = storage_entity.zip_path
+        klass = storage_entity.__class__.__name__
+        if not path.exists():
+            raise errors.PortalAPIError(400, f"Requested path does not exist: {path}")
+        log.debug(f"[SEND] Sending zip file {klass} "
+                  f"({self.submission.id}): {path}")
+        return flask.send_file(str(path), attachment_filename=path.name)
+
+    def send_selected_file(self, storage_entity: entities.Submission, path_query: str):
+        storage_entity = storage_entity or self.storage_entity
+        klass = storage_entity.__class__.__name__
+        log_name = self.submission.log_name if self.submission else self.project.log_name
+        log.debug(f"[SEND] Sending file {klass} {log_name}: {path_query}")
+        path = storage_entity.get(path_query)
+        return flask.send_file(str(path), attachment_filename=path.name)
+
+    def send_file_or_zip(self, storage_entity=None):
+        storage_entity = storage_entity or self.storage_entity
+        path_query = self.request.args.get('path')
+        if path_query is None:
+            return self.send_zip(storage_entity)
+        return self.send_selected_file(storage_entity, path_query)
+
+    def send_files_tree(self, storage_entity=None):
+        storage_entity = storage_entity or self.storage_entity
+        tree = storage_entity.tree()
+        log.debug(f"[TREE] Tree for the storage entity {storage_entity.entity_id}: {tree}")
+        return tree
+
+    def get_test_files_entity_from_storage(self, project=None):
+        project = project or self.project
+        if not project.config.test_files_commit_hash:
+            log.warning(f"Test files are not present "
+                        f"for the project {project.log_name}")
+            self._rest_service.projects(project).update_project_test_files()
+            self.__wait_for_test_files()
+        return storage.test_files.get(project.id)
+
+    def __wait_for_test_files(self):
+        project = self.project
+        wait_interval = 60
+        log.debug("[WAIT] Waiting for test files to be present")
+        while True:
+            self.refresh(project)
+            wait_interval -= 1
+            if wait_interval <= 0:
+                raise TimeoutError("Waiting for the test files timed out")
+            log.debug(f"[WAIT] Project Files Hash: {project.config.test_files_commit_hash}")
+            if project.config.test_files_commit_hash is not None:
+                break
+            time.sleep(1)

portal/service/submissions.py

@@ -5,16 +5,14 @@ Submissions service
 import logging
 import time
 from pathlib import Path
-from typing import Union, List
+from typing import List, Union
 
-import flask
 from celery.result import AsyncResult
-from storage import entities
 from werkzeug.utils import secure_filename
 
 from portal import storage
 from portal.async_celery import submission_processor, tasks
-from portal.database.models import Project, Submission, SubmissionState, User, Worker, Role, Group
+from portal.database.models import Group, Project, Role, Submission, SubmissionState, User, Worker
 from portal.rest.rest_helpers import FlaskRequestHelper
 from portal.service import errors
 from portal.service.general import GeneralService
@@ -93,10 +91,6 @@ class SubmissionsService(GeneralService):
     def storage(self):
         return storage
 
-    @property
-    def storage_submission(self):
-        return storage.submissions.get(self.submission.id)
-
     def process_new_submission(self) -> AsyncResult:
         project = self.submission.project
         self.submission.parameters['file_params'] = project.config.file_whitelist
@@ -161,36 +155,6 @@ class SubmissionsService(GeneralService):
         processor = submission_processor.SubmissionProcessor(self.submission)
         processor.revoke_task()
 
-    def send_zip(self, storage_submission: entities.Submission):
-        storage_submission = storage_submission or self.storage_submission
-        path = storage_submission.zip_path
-        klass = self.storage_submission.__class__.__name__
-        if not path.exists():
-            raise errors.PortalAPIError(400, f"Requested path does not exist: {path}")
-        log.debug(f"[SEND] Sending zip file {klass} "
-                  f"({self.submission.id}): {path}")
-        return flask.send_file(str(path), attachment_filename=path.name)
-
-    def send_selected_file(self, storage_entity: entities.Submission, path_query: str):
-        storage_entity = storage_entity or self.storage_submission
-        klass = self.storage_submission.__class__.__name__
-        log.debug(f"[SEND] Sending file {klass} {self.submission.log_name}: {path_query}")
-        path = storage_entity.get(path_query)
-        return flask.send_file(str(path), attachment_filename=path.name)
-
-    def send_file_or_zip(self, storage_entity=None):
-        storage_entity = storage_entity or self.storage_submission
-        path_query = self.request.args.get('path')
-        if path_query is None:
-            return self.send_zip(storage_entity)
-        return self.send_selected_file(storage_entity, path_query)
-
-    def send_files_tree(self, storage_entity=None):
-        storage_entity = storage_entity or self.storage_submission
-        tree = storage_entity.tree()
-        log.debug(f"[TREE] Tree for the storage entity {storage_entity.entity_id}: {tree} ")
-        return tree
-
     def upload_results_to_storage(self):
         path = self.get_upload_file_path()
         task = tasks.upload_results_to_storage.delay(self.submission.id, path=str(path))
@@ -202,29 +166,6 @@ class SubmissionsService(GeneralService):
         path = upload_files_to_storage(file)
         return path
 
-    def get_test_files_entity_from_storage(self):
-        project = self.project
-        if not project.config.test_files_commit_hash:
-            log.warning(f"Test files are not present "
-                        f"for the project {project.log_name}")
-            self._rest_service.projects(project).update_project_test_files()
-            self.__wait_for_test_files()
-        return storage.test_files.get(project.id)
-
-    def __wait_for_test_files(self):
-        project = self.project
-        wait_interval = 60
-        log.debug("[WAIT] Waiting for test files to be present")
-        while True:
-            self.refresh(project)
-            wait_interval -= 1
-            if wait_interval <= 0:
-                raise TimeoutError("Waiting for the test files timed out")
-            log.debug(f"[WAIT] Project Files Hash: {project.config.test_files_commit_hash}")
-            if project.config.test_files_commit_hash is not None:
-                break
-            time.sleep(1)
-
     def filter_user_avail_submissions(self, query, roles: List[Role], groups: List[Group]):
         submissions = query.all()
         return [submission for submission in submissions
@@ -242,10 +183,14 @@ class SubmissionsService(GeneralService):
         projects = request_helper.args.projects()
         roles = request_helper.args.roles()
         groups = request_helper.args.groups()
+        state = request_helper.args.state()
 
         if user:
             query = query.filter(Submission.user == user)
 
+        if state:
+            query = query.filter(Submission.state == state)
+
         if course:
             query = query.filter(Submission.course == course)
 

tests/rest/test_submission_files.py

@@ -74,6 +74,15 @@ def test_submission_test_files_are_available(client, mocked_submission):
     assert response.data.decode('utf-8') == expected('test_main.c')
 
 
+def test_project_test_files_are_available(client, mocked_submission):
+    s = mocked_submission
+    url = f'/courses/{s.course.id}/projects/{s.project.id}/files?path=test_main.c'
+    response = utils.make_request(client, url)
+    assert response.status_code == 200
+    assert response.data
+    assert response.data.decode('utf-8') == expected('test_main.c')
+
+
 def test_submission_sources_tree(client, mocked_submission):
     s = mocked_submission
     response = utils.make_request(client, f'/submissions/{s.id}/files/sources/tree')

tests/utils/ent_mocker.py

@@ -47,7 +47,7 @@ class EntitiesMocker:
         return self.data.create_submission(project=project, user=user)
 
     def create_submission_storage(self, submission):
-        storage = self.rest.submissions.storage
+        storage = self.rest.storage.storage
 
         def __create_content(location: Entity, *files):
             path = location.path