Worker API has been removed (a9c5c378) · Commits · Kontr 2.0 / Portal API Backend

Pipfile

+0 −1

Original line number	Diff line number	Diff line
		@@ -25,7 +25,6 @@ celery = {version = "*", extras = ["auth", "redis", "yaml", "msgpack"]}
		mockredispy = "*"
		python-slugify = "*"
		coloredlogs = "*"
		worker-api = {editable = true, git = "https://gitlab.fi.muni.cz/grp-kontr2/worker-api.git", ref='master'}
		"ldap3" = "*"

		[dev-packages]

Pipfile.lock

+8 −13

Original line number	Diff line number	Diff line
		{
		"_meta": {
		"hash": {
		"sha256": "78ad58bc50300482ca022013499e01eb3facfde2657d9913a6060f2388cf9bf0"
		"sha256": "f94131fd9b9fb9983abd7a9804e6f01746cb6c56a644a15e4310ba395e0e868d"
		},
		"pipfile-spec": 6,
		"requires": {
		@@ -352,11 +352,11 @@
		},
		"marshmallow": {
		"hashes": [
		"sha256:276db2f676763649262cd957757298329b07a20be513c1880e2763a0523260bf",
		"sha256:485ac6ed0dff5e1af6ea1e3a54425a448968f581b065424c89a5375e4d4866fd"
		"sha256:24a76fdc8857344fe896faa99b589ce10f44c691ce511fee37775757d482f036",
		"sha256:ec7a8d0101c0175480684f8b6431ec52dc0b3e05bbd780643c2ee7df7735f148"
		],
		"index": "pypi",
		"version": "==2.15.6"
		"version": "==2.16.0"
		},
		"marshmallow-enum": {
		"hashes": [
		@@ -582,11 +582,6 @@
		"sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b"
		],
		"version": "==0.14.1"
		},
		"worker-api": {
		"editable": true,
		"git": "https://gitlab.fi.muni.cz/grp-kontr2/worker-api.git",
		"ref": "8b24f7c0290c6b3447692f081fb67a1df6868b73"
		}
		},
		"develop": {
		@@ -721,11 +716,11 @@
		},
		"py": {
		"hashes": [
		"sha256:06a30435d058473046be836d3fc4f27167fd84c45b99704f2fb5509ef61f9af1",
		"sha256:50402e9d1c9005d759426988a492e0edaadb7f4e68bcddfea586bc7432d009c6"
		"sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694",
		"sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6"
		],
		"markers": "python_version != '3.1.' and python_version != '3.3.' and python_version >= '2.7' and python_version != '3.2.' and python_version != '3.0.'",
		"version": "==1.6.0"
		"markers": "python_version != '3.0.' and python_version >= '2.7' and python_version != '3.1.' and python_version != '3.3.' and python_version != '3.2.'",
		"version": "==1.7.0"
		},
		"pylint": {
		"hashes": [

portal/async_celery/submission_processor.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -132,7 +132,7 @@ class SubmissionProcessor:
		return worker

		def execute_submission(self, worker: Worker):
		worker_client = tools.worker_client.WorkerClient(worker=worker)
		worker_client = self._rest.workers(worker).worker_client
		worker_client.execute_submission(self.submission)

		def _worker_not_available(self):

portal/service/workers.py

+7 −0

Original line number	Diff line number	Diff line
		"""
		Components service
		"""
		import portal.tools.worker_client

		from portal import logger
		from portal.database.models import Worker
		@@ -18,6 +19,12 @@ class WorkerService(GeneralService):
		def worker(self):
		return self._entity

		@property
		def worker_client(self) -> portal.tools.worker_client.WorkerClient:
		if not hasattr(self, '__worker_client'):
		setattr(self, '__worker_client', portal.tools.worker_client.WorkerClient(self._entity))
		return getattr(self, '__worker_client')

		def _set_data(self, entity: Worker, **data) -> Worker:
		allowed = ['name', 'url', 'tags', 'portal_secret', 'state']
		return self.update_entity(entity, data, allowed=allowed)

portal/tools/worker_client.py

+95 −11

Original line number	Diff line number	Diff line
		import worker_api.client
		import urllib.parse
		from typing import Union

		import requests

		from portal import logger

		log = logger.get_logger(__name__)


		def _parse_result(result: requests.Response) -> Union[dict, bytes]:
		try:
		return result.json()
		except ValueError:
		return result.content


		class WorkerClient:
		def __init__(self, worker):
		def __init__(self, url: str = None, secret: str = None, worker=None, name: str = None):
		self._worker = worker
		self._api = None
		self._url = url or worker.url
		self._secret = secret or worker.portal_secret
		self._session = None
		self._name = name

		@property
		def worker_name(self):
		if self.worker:
		return self.worker.log_name
		if self._name:
		return self._name
		if self._url:
		return self._url

		@property
		def _auth_header(self):
		return {'Authorization': f'Bearer {self._secret}'}

		@property
		def worker_url(self):
		return self._url

		@property
		def session(self) -> requests.Session:
		if not self._session:
		self._session = requests.session()
		self._session.headers.update(self._auth_header)
		return self._session

		@property
		def worker(self):
		@@ -22,16 +59,63 @@ class WorkerClient:
		)
		return cred

		@property
		def api(self) -> worker_api.client.WorkerClient:
		if self._api is None:
		self._api = worker_api.client.WorkerClient(**self.credentials)
		return self._api
		def make_request(self, path, method='get', **kwargs):
		url = self._get_url(path)
		log.debug(f"[W-REQ] ({method}) {url}")
		result = self.session.request(url=url, method=method, **kwargs)
		if not result.ok:
		log.warning(f"[W-API] Worker returned the error "
		f"({result.status_code}): {result.content}")
		else:

		log.debug(f"[W-RES] ({result.status_code}) {result.content}")
		return _parse_result(result)

		def execute_submission(self, submission):
		log.info(f"[WC] Executing: {submission.log_name}")
		log.info(f"[W-API] Executing in {self.worker_name}: {submission.log_name}")
		parameters = submission.parameters
		parameters['test_files_hash'] = submission.project.config.test_files_commit_hash
		parameters['id'] = submission.id
		result = self.api.submissions.create(parameters, sid=submission.id)
		log.debug(f"[WC] Exec call result: {result}")
		result = self.make_request(f'/submissions/{submission.id}', method='post', json=parameters)
		return result

		def get_submission(self, submission):
		log.info(f"[W-API] Get submission in {self.worker_name}: {submission.log_name}")
		result = self.make_request(f'/submissions/{submission.id}', method='get')
		log.debug(f"[W-API] Submission: {result}")
		return result

		def list_submissions(self):
		log.info(f"[W-API] Get submissions for {self.worker_name}")
		result = self.make_request(f'/submissions', method='get')
		log.debug(f"[W-API] Submission: {result}")
		return result

		def status(self):
		log.info(f"[W-API] Get status for {self.worker_name}")
		result = self.make_request(f'/management/status', method='get')
		log.debug(f"[W-API] Status: {result}")
		return result

		def list_images(self, ):
		log.info(f"[W-API] Get images for {self.worker_name}")
		result = self.make_request(f'/submissions', method='get')
		log.debug(f"[W-API] Images: {result}")
		return result

		def remove_image(self, project):
		commit_hash = project.config.test_files_commit_hash
		log.info(f"[W-API] Removing image from test files in {self.worker_name} for "
		f"{project.log_name}: {commit_hash}")
		result = self.make_request(f'/images/{commit_hash}/clean', method='get')
		log.debug(f"[W-API] State: {result}")
		return result

		def register(self):
		log.info(f"[W-API] Register worker {self.worker_name}")
		result = self.make_request(f'/management/registration', method='post')
		log.debug(f"[W-API] State: {result}")
		return result

		def _get_url(self, path: str):
		return urllib.parse.urljoin(self.worker_url, '/api/v1.0', path)