Verified Commit 829b5286 authored by Peter Stanko's avatar Peter Stanko
Browse files

Storage moved and merged to a portal

parent cb89ca8f
......@@ -9,7 +9,6 @@ flask-sqlalchemy = "*"
marshmallow = "*"
flask-jwt-extended = "*"
marshmallow-enum = "*"
storage = {editable = true,ref = "master",git = "https://gitlab.fi.muni.cz/grp-kontr2/kontr-storage-module.git"}
gitpython = "*"
pyopenssl = "*"
gunicorn = "*"
......
This diff is collapsed.
......@@ -10,7 +10,7 @@ from pathlib import Path
from flask_sqlalchemy import SQLAlchemy
from portal import logger, storage
from portal import logger, storage_wrapper
from portal.database.models import Course, Group, Project, ReviewItem, Role, Submission, User, \
Worker
from portal.tools import time
......@@ -160,16 +160,16 @@ class DataFactory(object):
log.debug(f"[CREATE] Project config: {project.log_name}: {config}")
project.set_config(**config)
self.session.flush()
test_path = storage.test_files.path / project.id
test_path = storage_wrapper.test_files.path / project.id
shutil.copytree(str(TESTS_DIR), str(test_path))
return project
def create_submission(self, project, user, **kwargs):
submission = self.__create_entity(Submission, project=project, user=user, **kwargs)
self.session.flush()
subm_path = storage.submissions.path / submission.id
subm_path = storage_wrapper.submissions.path / submission.id
shutil.copytree(str(SOURCES_DIR), str(subm_path))
res_path = storage.results.path / submission.id
res_path = storage_wrapper.results.path / submission.id
shutil.copytree(str(RESULTS_DIR), str(res_path))
return submission
......
......@@ -12,7 +12,7 @@ from flask_jwt_extended import JWTManager
from flask_migrate import Migrate
from authlib.flask.client import OAuth
from flask_sqlalchemy import SQLAlchemy
from storage import Storage
from portal.storage import Storage
from portal import logger, rest
from portal.config import CONFIGURATIONS
......@@ -26,7 +26,7 @@ db = SQLAlchemy()
jwt = JWTManager()
oauth = OAuth()
cors = CORS(supports_credentials=True)
storage = Storage()
storage_wrapper = Storage()
migrate = Migrate(db=db)
gitlab_factory = GitlabFactory()
ldap_wrapper = LDAPWrapper()
......@@ -86,7 +86,7 @@ def configure_storage(app: Flask) -> Flask:
workspace_dir=app.config.get('PORTAL_STORAGE_WORKSPACE_DIR'),
results_dir=app.config.get('PORTAL_STORAGE_RESULTS_DIR')
)
storage.init_storage(**storage_config)
storage_wrapper.init_storage(**storage_config)
return app
......
......@@ -3,12 +3,11 @@ import secrets
from pathlib import Path
from typing import Optional
from storage import UploadedEntity, entities
from portal import logger
from portal.database import Project, Submission, SubmissionState, Worker
from portal.database.models import WorkerState
from portal.service import errors
from portal.storage import UploadedEntity, entities
log = logger.get_logger(__name__)
......@@ -39,8 +38,8 @@ class SubmissionProcessor:
@property
def storage(self):
from portal import storage
return storage
from portal import storage_wrapper
return storage_wrapper
def reset_task_id(self, state=None):
if state is not None:
......
from celery.utils.log import get_task_logger
from storage import UploadedEntity
from portal.storage import UploadedEntity
from portal import storage
from portal import storage_wrapper
from portal.async_celery import celery_app, submission_processor
from portal.service.find import FindService
from portal.service.general import GeneralService
......@@ -93,7 +93,7 @@ def is_import_users_for_course(course_id: str, role_name: str, users_type: str):
def _update_test_files(project, params):
updated_entity: UploadedEntity = storage.test_files.update(entity_id=project.id, **params)
updated_entity: UploadedEntity = storage_wrapper.test_files.update(entity_id=project.id, **params)
project.config.test_files_commit_hash = updated_entity.version
log.debug(f"Updated project config {project.log_name}: {project.config}")
GeneralService().write_entity(project.config)
......
......@@ -67,8 +67,8 @@ class GeneralFacade:
@property
def storage(self):
from portal import storage
return storage
from portal import storage_wrapper
return storage_wrapper
def _require_params(self, dictionary, *params):
for param in params:
......
......@@ -103,7 +103,7 @@ class Logging:
'werkzeug': {
'handlers': ['console'], 'level': self.global_log_level, 'propagate': True
},
'storage': {
'portal.storage': {
'handlers': ['console', 'storage_file'], 'level': self.global_log_level,
'propagate': True
},
......
import flask
import storage
import portal.storage
from flask import Flask
from flask_jwt_extended.exceptions import NoAuthorizationError
from marshmallow.exceptions import ValidationError
......@@ -60,19 +60,19 @@ def handle_db_error(ex: SQLAlchemyError):
@rest_api.errorhandler(NotImplementedError)
def handle_not_implemented_error():
def handle_not_implemented_error(ex: NotImplementedError):
log.warning(f"[WARN] Not implemented yet: {ex}")
return send_response({'message': f'Not implemented yet!'}), 404
@rest_api.errorhandler(storage.errors.NotFoundError)
def handle_storage_not_found_error(ex: storage.errors.NotFoundError):
@rest_api.errorhandler(portal.storage.errors.NotFoundError)
def handle_storage_not_found_error(ex: portal.storage.errors.NotFoundError):
log.warning(f"[STORAGE] Storage not found warning: {ex}")
return send_response({'message': str(ex)}), 404
@rest_api.errorhandler(storage.errors.KontrStorageError)
def handle_storage_error(ex: storage.errors.KontrStorageError):
@rest_api.errorhandler(portal.storage.errors.KontrStorageError)
def handle_storage_error(ex: portal.storage.errors.KontrStorageError):
log.warning(f"[STORAGE] Storage error: {ex}")
return send_response({'message': str(ex)}), 400
......
......@@ -2,9 +2,9 @@ import time
from typing import Optional
import flask
from storage import entities
from portal.storage import entities, Storage
from portal import logger, storage
from portal import logger, storage_wrapper
from portal.database import Project, Submission
from portal.service import errors
from portal.service.general import GeneralService
......@@ -17,8 +17,8 @@ class StorageService(GeneralService):
return None
@property
def storage(self):
return storage
def storage(self) -> Storage:
return storage_wrapper
def __call__(self, submission=None, project=None):
self._project = project
......@@ -87,7 +87,7 @@ class StorageService(GeneralService):
from .projects import ProjectService
ProjectService(project).update_project_test_files()
self.__wait_for_test_files()
return storage.test_files.get(project.id)
return self.storage.test_files.get(project.id)
def __wait_for_test_files(self):
project = self.project
......
......@@ -9,7 +9,7 @@ from pathlib import Path
from celery.result import AsyncResult
from werkzeug.utils import secure_filename
from portal import storage
from portal import storage_wrapper
from portal.async_celery import submission_processor, tasks
from portal.database import queries
from portal.database.models import Project, Submission, User
......@@ -21,7 +21,7 @@ log = logging.getLogger(__name__)
def upload_files_to_storage(file):
filename = secure_filename(file.filename)
uploads = storage.results.workspace_path / 'uploads'
uploads = storage_wrapper.results.workspace_path / 'uploads'
if not uploads.exists():
uploads.mkdir(parents=True)
path = uploads / filename
......@@ -100,7 +100,7 @@ class SubmissionsService(GeneralService):
@property
def storage(self):
return storage
return storage_wrapper
def process_new_submission(self) -> AsyncResult:
project = self.submission.project
......@@ -146,11 +146,11 @@ class SubmissionsService(GeneralService):
# TODO: dispatch action to clean the storage
super(SubmissionsService, self).delete()
log.info(f"[DEL] Submission delete submission files: {self.submission.log_name}")
st_entity = storage.submissions.get(self.submission.id)
st_entity = storage_wrapper.submissions.get(self.submission.id)
if st_entity:
st_entity.delete()
log.info(f"[DEL] Submission delete result files: {self.submission.log_name}")
st_entity = storage.results.get(self.submission.id)
st_entity = storage_wrapper.results.get(self.submission.id)
if st_entity:
st_entity.delete()
......@@ -230,11 +230,11 @@ class SubmissionsService(GeneralService):
def archive(self):
log.info(f"[ARCHIVE] Submission archive submission files: {self.submission.log_name}")
st_entity = storage.submissions.get(self.submission.id)
st_entity = storage_wrapper.submissions.get(self.submission.id)
if st_entity:
st_entity.clean()
log.info(f"[ARCHIVE] Submission archive result files: {self.submission.log_name}")
st_entity = storage.results.get(self.submission.id)
st_entity = storage_wrapper.results.get(self.submission.id)
if st_entity:
st_entity.clean()
......
from .storage import Storage
from .entities import UploadedEntity, Submission, Entity, TestFiles
from .errors import NotFoundError, KontrStorageError
import logging
import shutil
from pathlib import Path
from typing import Union
from portal.storage import errors, utils
from portal.storage.errors import InvalidPath
from portal.storage.inputs import filter_entity, git_clone
from portal.storage.inputs.zip import unzip_entity
from portal.storage.utils import delete_dir, get_directory_structure
from portal.storage.utils.compress import compress_entity, decompress_entity
from portal.storage.utils.update import entity_update
log = logging.getLogger(__name__)
class Entity(object):
def __init__(self, entity_id: str, base_dir: Union[Path, str]):
"""Creates a storage entity
Args:
entity_id(str): Id of the entity
base_dir(Path, str): Base dir for the entity
"""
self._base_dir = Path(base_dir)
self._entity_id = entity_id
@property
def entity_id(self) -> str:
"""Gets an entity id
Returns(str): Entity id
"""
return self._entity_id
@property
def base_dir(self) -> Path:
"""Gets an base dir where the all entities are being created
Returns(Path): Base dir of all entities
"""
return self._base_dir
@property
def path(self) -> Path:
"""Gets a path to the entity
Returns(Path): Entity id
"""
full_path = self.base_dir / self.entity_id
return full_path
def get(self, path_query: str):
"""Gets file based on relative path query
Examples:
path_query = "src/main.c"
root = "/storage/submission"
Result: "/storage/submission/src/main.c"
Args:
path_query(str): Path query
Returns(Path): Absolute path to requested file
"""
path_query = Path(path_query)
if not utils.is_forward_path(self.path, path_query):
log.error(f"Not a valid path: {path_query} for: {self.path}/#{path_query}")
raise InvalidPath(f"Not a valid path: {path_query} for: {self.path}/#{path_query}")
path = (self.path / path_query)
abs_path = path.absolute()
if not abs_path.exists():
log.error(f"Path {abs_path} not exists!")
raise errors.NotFoundError(f"Path {abs_path} not exists!")
return path
@property
def zip_path(self) -> Path:
"""Gets a path to the zip file of the entity
Returns(Path): Zip file path
"""
return self.base_dir / f"{self.entity_id}.zip"
def subdir(self, subdir: str) -> Path:
"""Gets an subdirectory from the path
Args:
subdir(str): Subdirectory
Returns(Path): Full path
"""
return self.path / subdir
def compress(self) -> Path:
"""Compresses the entity
Returns(Path): Path to the zip file
"""
return compress_entity(self)
def decompress(self) -> Path:
"""Decompress the entity
Returns(Path): The entity directory path
"""
return decompress_entity(self)
def clean(self):
self.compress()
shutil.rmtree(str(self.path))
def update_subdir(self, source: Union[str, Path], subdir: str) -> Path:
"""Updates subdirectory of the entity with files
Args:
source(str, Path): Source subdir
subdir(str): Subdir
Returns(Path):
"""
destination = self.subdir(subdir=subdir)
entity_update(src=source, dst=destination)
return destination
def content(self, file_path: Union[Path, str], subdir='') -> str:
"""Gets an content of the file
Args:
file_path(Path,str): Path to the file
subdir(str): Subdirectory
Returns(str): String content of the file
"""
full_path = self.subdir(subdir=subdir) / file_path
with full_path.open('r') as content_file:
return content_file.read()
def tree(self, subdir='') -> dict:
"""Gets a tree representation of file structure of the submission
Args:
subdir(str): From which subdir
Returns(dict):
"""
return get_directory_structure(self.subdir(subdir))
def glob(self, pattern: str, subdir=''):
"""Find files in the entity path
Args:
pattern(str): Pattern to find in the directory
subdir(str): Subdirectory
Returns: Iterable
"""
full = self.subdir(subdir=subdir)
return full.glob(pattern=pattern)
def delete(self):
if self.zip_path.exists():
self.zip_path.unlink()
if self.path.exists():
delete_dir(self.path)
class TestFiles(Entity):
pass
class Submission(Entity):
pass
class Results(Entity):
pass
class UploadedEntity(object):
"""Working entity - upload process
"""
def __init__(self, entity: Entity, workspace: Path, **kwargs):
self._entity = entity
self._params = {**kwargs}
self._workspace = workspace
@property
def entity(self) -> Entity:
return self._entity
@property
def workspace(self) -> Path:
return self._workspace
@property
def path(self) -> Path:
return self.entity.path
@property
def path(self) -> Path:
return self.entity.path
@property
def type(self) -> str:
return self.config.get('type')
@property
def filters(self) -> str:
return self._params.get('whitelist') or '*'
@property
def from_dir(self) -> Path:
return self._params.get('from_dir') or ''
@property
def config(self) -> dict:
return self._params.get('source')
@property
def version(self) -> str:
return self._params.get('version')
@version.setter
def version(self, value: str):
self._params['version'] = value
@property
def additional(self) -> str:
return self._params.get('additional')
@additional.setter
def additional(self, value):
self._params['additional'] = value
def download(self):
if self.type == 'git':
return git_clone(self)
elif self.type == 'zip':
return unzip_entity(self)
else:
return None
def filter(self):
return filter_entity(self)
def clean(self):
if self.workspace.exists():
return delete_dir(self.workspace)
def process(self) -> 'UploadedEntity':
self.download()
self.filter()
self.clean()
self.entity.compress()
return self
class KontrStorageError(RuntimeError):
pass
class NotFoundError(KontrStorageError):
pass
class InvalidPath(KontrStorageError):
pass
\ No newline at end of file
from .git import GitWrapper, git_clone
from .filter import filter_entity
import logging
from portal.storage.utils import clone_files
log = logging.getLogger(__name__)
def filter_entity(entity):
filters = entity.filters.splitlines()
entity.path.mkdir(parents=True)
from_path = entity.workspace / entity.from_dir
for pattern in filters:
log.info(f"[FLT] Filtering using filter for {pattern}")
clone_files(src=from_path, dst=entity.path, pattern=pattern)
return entity
import logging
from git import Repo
log = logging.getLogger(__name__)
def git_clone(entity):
"""Clones the repository
Args:
entity: Entity instance
Returns(Entity): Entity instance
"""
return GitWrapper(entity=entity).clone()
class GitEntityConfig(object):
def __init__(self, entity):
self.entity = entity
@property
def config(self) -> dict:
"""Entity config
Returns(dict): Entity config
"""
return self.entity.config
@property
def url(self) -> str:
"""Git repo url
Returns(str): Repository url
"""
return self.config['url']
@property
def branch(self) -> str:
"""Branch name
Returns(str): Branch name
"""
return self.config.get('branch')
@property
def checkout(self) -> str:
"""Checkout (commit, branch, tag)
Returns(std): Which version to checkout
"""
return self.config.get('checkout')
@property
def effective_checkout(self) -> str:
"""Effective checkout - whether to checkout branch or tag or commit
Returns(str): Checkout name
"""
return self.checkout or self.branch
class GitWrapper(object):
def __init__(self, entity):
self.entity = entity
self.config = GitEntityConfig(self.entity)
self._repo = None
@property