Autodownload test files for the new submission

portal/async_celery/tasks.py

@@ -11,6 +11,10 @@ log = get_task_logger(__name__)
 @celery_app.task(name='upload-submission-to-storage')
 def process_submission(new_submission_id: str):
     new_submission = find_submission(new_submission_id)
+    project = new_submission.project
+    course = project.course
+    if not project.config.test_files_commit_hash:
+        update_project_test_files(course_id=course.id, project_id=project.id)
     processor = submission_processor.SubmissionProcessor(new_submission)
     processor.process_submission()
 

portal/rest/submissions.py

+import time
+
 import flask
 from flask_jwt_extended import jwt_required
 from flask_restplus import Namespace, Resource
 
 from portal import logger, storage
+from portal.database import Project
 from portal.rest import rest_helpers
 from portal.rest.schemas import SCHEMAS
 from portal.service import auth, general, permissions
+from portal.service.projects import ProjectService
 from portal.service.reviews import ReviewService
 from portal.service.submissions import SubmissionsService
 
@@ -113,8 +117,8 @@ class SubmissionTestFilesTree(Resource):
         submission = general.find_submission(sid)
         course = submission.project.course
         permissions.PermissionsService(course=course).require.read_submission_group(submission)
-        storage_entity = storage.test_files.get(submission.project.id)
         service = SubmissionsService(submission=submission)
+        storage_entity = service.get_test_files_entity_from_storage()
         return service.send_files_tree(storage_entity)
 
 
@@ -127,8 +131,8 @@ class SubmissionTestFiles(Resource):
         submission = general.find_submission(sid)
         course = submission.project.course
         permissions.PermissionsService(course=course).require.read_submission_group(submission)
-        storage_entity = storage.test_files.get(submission.project.id)
         service = SubmissionsService(submission=submission)
+        storage_entity = service.get_test_files_entity_from_storage()
         return service.send_file_or_zip(storage_entity)
 
 
@@ -243,3 +247,4 @@ class SubmissionReview(Resource):
         review_service.create_review_items(items=data['review_items'], author=client)
 
         return SCHEMAS.dump('review', submission.review), 201
+

portal/service/general.py

@@ -58,6 +58,29 @@ def write_entity(entity):
     db.session.commit()
 
 
+def refresh(obj) -> object:
+    """Refreshes the object
+    Args:
+        obj: Any db model
+    Returns: Instance of the same object
+
+    """
+    log.debug(f"[REFRESH] Entity: {obj}")
+    db.session.refresh(obj)
+    return obj
+
+
+def expire(obj) -> object:
+    """Expires the object attributes
+    Args:
+        obj: Any db model
+    Returns: Instance of the same object
+    """
+    log.debug(f"[EXPIRE] Entity: {obj}")
+    db.session.expire(obj)
+    return obj
+
+
 def find_resource(identifier: str, resource: str, query, throws: bool = True):
     """Gets an instance of the resource
 

portal/service/submissions.py

@@ -3,6 +3,7 @@ Submissions service
 """
 
 import logging
+import time
 from pathlib import Path
 from typing import Union
 
@@ -14,8 +15,9 @@ from werkzeug.utils import secure_filename
 from portal import storage
 from portal.async_celery import submission_processor, tasks
 from portal.database.models import Project, Submission, SubmissionState, User, Worker
-from portal.service import errors
+from portal.service import errors, general
 from portal.service.general import delete_entity, write_entity
+from portal.service.projects import ProjectService
 
 log = logging.getLogger(__name__)
 
@@ -193,3 +195,26 @@ class SubmissionsService(object):
         upload_file_is_allowed(file)
         path = upload_files_to_storage(file)
         return path
+
+    def get_test_files_entity_from_storage(self):
+        project = self.project
+        if not project.config.test_files_commit_hash:
+            log.warning(f"Test files are not present "
+                        f"for the project {project.id} ({project.codename})")
+            ProjectService(project=project).update_project_test_files()
+            self.__wait_for_test_files()
+        return storage.test_files.get(project.id)
+
+    def __wait_for_test_files(self):
+        project = self.project
+        wait_interval = 60
+        log.debug("[WAIT] Waiting for test files to be present")
+        while True:
+            general.refresh(project)
+            wait_interval -= 1
+            if wait_interval <= 0:
+                raise TimeoutError("Waiting for the test files timed out")
+            log.debug(f"[WAIT] Project Files Hash: {project.config.test_files_commit_hash}")
+            if project.config.test_files_commit_hash is not None:
+                break
+            time.sleep(1)