25 Add source_hash to Submission & test_files_commit_hash to ProjectConfig (98003104) · Commits · Kontr 2.0 / Portal API Backend

migrations/versions/9143d16ae835_.py

0 → 100644

+30 −0

Original line number	Original line	Diff line number	Diff line
			"""empty message

			Revision ID: 9143d16ae835
			Revises: e897fc93ab4e
			Create Date: 2018-08-25 13:07:56.545852

			"""
			from alembic import op
			import sqlalchemy as sa


			# revision identifiers, used by Alembic.
			revision = '9143d16ae835'
			down_revision = 'e897fc93ab4e'
			branch_labels = None
			depends_on = None


			def upgrade():
			# ### commands auto generated by Alembic - please adjust! ###
			op.add_column('projectConfig', sa.Column('test_files_commit_hash', sa.String(length=64), nullable=True))
			op.add_column('submission', sa.Column('source_hash', sa.String(length=64), nullable=True))
			# ### end Alembic commands ###


			def downgrade():
			# ### commands auto generated by Alembic - please adjust! ###
			op.drop_column('submission', 'source_hash')
			op.drop_column('projectConfig', 'test_files_commit_hash')
			# ### end Alembic commands ###

portal/async_celery/storage.py

+2 −1

Original line number	Original line	Diff line number	Diff line
	@@ -7,9 +7,10 @@ from portal.database.models import Submission, SubmissionState
	log = logging.getLogger(__name__)		log = logging.getLogger(__name__)


	def submission_store_ended(submission: Submission):		def submission_store_ended(submission: Submission, version: str):
	# TODO: TBD		# TODO: TBD
	log.info(f"[ASYNC] Submission preparation ended: {submission}")		log.info(f"[ASYNC] Submission preparation ended: {submission}")
			submission.source_hash = version
	reset_task_id(submission=submission, task_id_type='storage_task_id',		reset_task_id(submission=submission, task_id_type='storage_task_id',
	state=SubmissionState.READY)		state=SubmissionState.READY)

portal/async_celery/tasks.py

+22 −5

Original line number	Original line	Diff line number	Diff line
	from pathlib import Path		from pathlib import Path

	from celery.utils.log import get_task_logger		from celery.utils.log import get_task_logger
			from storage import UploadedEntity

	from portal import storage		from portal import storage
	from portal.async_celery import celery_app		from portal.async_celery import celery_app
	from portal.async_celery.storage import submission_store_ended		from portal.async_celery.storage import submission_store_ended
	from portal.database import SubmissionState		from portal.database import SubmissionState
	from portal.service import general		from portal.service import general
	from portal.service.general import find_submission		from portal.service.general import find_submission, find_project, find_course, write_entity

	log = get_task_logger(__name__)		log = get_task_logger(__name__)

	@@ -17,9 +18,9 @@ def upload_submission_to_storage(new_submission_id: str, file_params: dict):
	new_submission = find_submission(new_submission_id)		new_submission = find_submission(new_submission_id)
	log.info(		log.info(
	f"[ASYNC] Uploading submission: {new_submission} with {file_params}")		f"[ASYNC] Uploading submission: {new_submission} with {file_params}")
	storage.submissions.create(entity_id=new_submission.id, **file_params)		updated_entity: UploadedEntity = storage.submissions.create(entity_id=new_submission.id, **file_params)
	# TODO: Upload ended -> change state		# TODO: Upload ended -> change state
	submission_store_ended(submission=new_submission)		submission_store_ended(submission=new_submission, version=updated_entity.version)


	@celery_app.task(name='upload-results-to-storage')		@celery_app.task(name='upload-results-to-storage')
	@@ -35,14 +36,13 @@ def upload_results_to_storage(new_submission_id: str, path: str):
	general.write_entity(new_submission)		general.write_entity(new_submission)



	@celery_app.task(name='clone-submission-files')		@celery_app.task(name='clone-submission-files')
	def clone_submission_files(source_id: str, target_id: str):		def clone_submission_files(source_id: str, target_id: str):
	source = find_submission(source_id)		source = find_submission(source_id)
	target = find_submission(target_id)		target = find_submission(target_id)
	log.info(f"[ASYNC] Cloning submission: {source} to {target}")		log.info(f"[ASYNC] Cloning submission: {source} to {target}")
	storage.submissions.clone(source.id, target.id)		storage.submissions.clone(source.id, target.id)
	submission_store_ended(target)		submission_store_ended(target, version=source.source_hash)


	@celery_app.task(name='start-processing-submission')		@celery_app.task(name='start-processing-submission')
	@@ -52,3 +52,20 @@ def start_processing_submission(submission_id: str, submission_params):
	# TODO: implement processing		# TODO: implement processing
	if submission_params is not None:		if submission_params is not None:
	print("Not implemented")		print("Not implemented")


			@celery_app.task(name='update-project-test-files')
			def update_project_test_files(course_id: str, project_id: str):
			log.info(f"[ASYNC] Updating test files for project: {project_id}")
			course = find_course(course_id)
			project = find_project(course, project_id)
			params = {
			'source': {
			'type': 'git',
			'url': project.config.test_files_source
			}
			}
			updated_entity: UploadedEntity = storage.test_files.update(entity_id=project.id, **params)
			project.config.test_files_commit_hash = updated_entity.version

			write_entity(project)

portal/database/models.py

+4 −0

portal/rest/projects.py

+20 −1

Original line number	Original line	Diff line number	Diff line
	@@ -12,7 +12,7 @@ from portal.service import auth, general
	from portal.service.errors import ForbiddenError, SubmissionRefusedError		from portal.service.errors import ForbiddenError, SubmissionRefusedError
	from portal.service.permissions import check_client, require_client		from portal.service.permissions import check_client, require_client
	from portal.service.projects import can_create_submission, create_project, delete_project, \		from portal.service.projects import can_create_submission, create_project, delete_project, \
	find_project_submissions, list_projects, update_project, update_project_config		find_project_submissions, list_projects, update_project, update_project_config, update_project_test_files_hash
	from portal.service.submissions import create_submission		from portal.service.submissions import create_submission
	from portal.tools import time		from portal.tools import time

	@@ -139,6 +139,25 @@ class ProjectConfigResource(Resource):
	return '', 204		return '', 204


			@projects_namespace.route('/courses/<string:cid>/projects/<string:pid>/test-files-refresh')
			@projects_namespace.param('cid', 'Course id')
			@projects_namespace.param('pid', 'Project id')
			@projects_namespace.response(404, 'Course not found')
			@projects_namespace.response(404, 'Project not found')
			class ProjectTestFilesRefresh(Resource):
			@jwt_required
			@projects_namespace.response(204, 'Project test_files updated')
			def post(self, cid: str, pid: str):
			client = auth.find_client()
			course = general.find_course(cid)
			project = general.find_project(course, pid)
			# authorization
			perm = ['write_projects', 'update_course']
			require_client(client=client, course=course, permissions=perm)
			update_project_test_files_hash(project)
			return '', 204


	@projects_namespace.route(		@projects_namespace.route(
	'/courses/<string:cid>/projects/<string:pid>/submissions')		'/courses/<string:cid>/projects/<string:pid>/submissions')
	@projects_namespace.param('cid', 'Course id')		@projects_namespace.param('cid', 'Course id')

Original line number	Original line	Diff line number	Diff line
	@@ -468,6 +468,7 @@ class ProjectConfig(db.Model, EntityBase):
	project_id: Associated project's id		project_id: Associated project's id
	project: Associated project		project: Associated project
	test_files_source: Test files source location: git repository		test_files_source: Test files source location: git repository
			test_files_commit_hash: Full SHA-1 hash of the latest revision of project's test_files
	file_whitelist: whitelisted files		file_whitelist: whitelisted files
	pre_submit_script: Pre submit script used in the submissions processing component		pre_submit_script: Pre submit script used in the submissions processing component
	post_submit_script: Post submit script used in the submissions processing component		post_submit_script: Post submit script used in the submissions processing component
	@@ -485,6 +486,7 @@ class ProjectConfig(db.Model, EntityBase):

	submissions_cancellation_period = db.Column(db.Integer)		submissions_cancellation_period = db.Column(db.Integer)
	test_files_source = db.Column(db.String(600)) # a git repository URL		test_files_source = db.Column(db.String(600)) # a git repository URL
			test_files_commit_hash = db.Column(db.String(64)) # hash of the latest revision of test_files
	file_whitelist = db.Column(db.Text)		file_whitelist = db.Column(db.Text)
	pre_submit_script = db.Column(db.Text)		pre_submit_script = db.Column(db.Text)
	post_submit_script = db.Column(db.Text)		post_submit_script = db.Column(db.Text)
	@@ -787,6 +789,7 @@ class Submission(db.Model, EntityBase):
	parameters: Parameters of the submission		parameters: Parameters of the submission
	state: State of the submission		state: State of the submission
	note: Notes with tags for the submission		note: Notes with tags for the submission
			source_hash: The git commit hash of the submitted repository revision

	user: User who created the submission		user: User who created the submission
	project: Associated project for the submission		project: Associated project for the submission
	@@ -801,6 +804,7 @@ class Submission(db.Model, EntityBase):
	state = db.Column(		state = db.Column(
	db.Enum(SubmissionState, name='SubmissionState'), nullable=False)		db.Enum(SubmissionState, name='SubmissionState'), nullable=False)
	note = db.Column(db.Text)		note = db.Column(db.Text)
			source_hash = db.Column(db.String(64))

	user_id = db.Column(db.String(36), db.ForeignKey(		user_id = db.Column(db.String(36), db.ForeignKey(
	'user.id', ondelete='cascade'), nullable=False)		'user.id', ondelete='cascade'), nullable=False)