Skip to content
Snippets Groups Projects
Verified Commit a9c5c378 authored by Peter Stanko's avatar Peter Stanko
Browse files

Worker API has been removed

parent 4525b390
No related branches found
No related tags found
1 merge request!39Cancelled submissions are not affecting the delay
......@@ -25,7 +25,6 @@ celery = {version = "*", extras = ["auth", "redis", "yaml", "msgpack"]}
mockredispy = "*"
python-slugify = "*"
coloredlogs = "*"
worker-api = {editable = true, git = "https://gitlab.fi.muni.cz/grp-kontr2/worker-api.git", ref='master'}
"ldap3" = "*"
[dev-packages]
......
{
"_meta": {
"hash": {
"sha256": "78ad58bc50300482ca022013499e01eb3facfde2657d9913a6060f2388cf9bf0"
"sha256": "f94131fd9b9fb9983abd7a9804e6f01746cb6c56a644a15e4310ba395e0e868d"
},
"pipfile-spec": 6,
"requires": {
......@@ -352,11 +352,11 @@
},
"marshmallow": {
"hashes": [
"sha256:276db2f676763649262cd957757298329b07a20be513c1880e2763a0523260bf",
"sha256:485ac6ed0dff5e1af6ea1e3a54425a448968f581b065424c89a5375e4d4866fd"
"sha256:24a76fdc8857344fe896faa99b589ce10f44c691ce511fee37775757d482f036",
"sha256:ec7a8d0101c0175480684f8b6431ec52dc0b3e05bbd780643c2ee7df7735f148"
],
"index": "pypi",
"version": "==2.15.6"
"version": "==2.16.0"
},
"marshmallow-enum": {
"hashes": [
......@@ -582,11 +582,6 @@
"sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b"
],
"version": "==0.14.1"
},
"worker-api": {
"editable": true,
"git": "https://gitlab.fi.muni.cz/grp-kontr2/worker-api.git",
"ref": "8b24f7c0290c6b3447692f081fb67a1df6868b73"
}
},
"develop": {
......@@ -721,11 +716,11 @@
},
"py": {
"hashes": [
"sha256:06a30435d058473046be836d3fc4f27167fd84c45b99704f2fb5509ef61f9af1",
"sha256:50402e9d1c9005d759426988a492e0edaadb7f4e68bcddfea586bc7432d009c6"
"sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694",
"sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6"
],
"markers": "python_version != '3.1.*' and python_version != '3.3.*' and python_version >= '2.7' and python_version != '3.2.*' and python_version != '3.0.*'",
"version": "==1.6.0"
"markers": "python_version != '3.0.*' and python_version >= '2.7' and python_version != '3.1.*' and python_version != '3.3.*' and python_version != '3.2.*'",
"version": "==1.7.0"
},
"pylint": {
"hashes": [
......
......@@ -132,7 +132,7 @@ class SubmissionProcessor:
return worker
def execute_submission(self, worker: Worker):
worker_client = tools.worker_client.WorkerClient(worker=worker)
worker_client = self._rest.workers(worker).worker_client
worker_client.execute_submission(self.submission)
def _worker_not_available(self):
......
"""
Components service
"""
import portal.tools.worker_client
from portal import logger
from portal.database.models import Worker
......@@ -18,6 +19,12 @@ class WorkerService(GeneralService):
def worker(self):
return self._entity
@property
def worker_client(self) -> portal.tools.worker_client.WorkerClient:
if not hasattr(self, '__worker_client'):
setattr(self, '__worker_client', portal.tools.worker_client.WorkerClient(self._entity))
return getattr(self, '__worker_client')
def _set_data(self, entity: Worker, **data) -> Worker:
allowed = ['name', 'url', 'tags', 'portal_secret', 'state']
return self.update_entity(entity, data, allowed=allowed)
......
import worker_api.client
import urllib.parse
from typing import Union
import requests
from portal import logger
log = logger.get_logger(__name__)
def _parse_result(result: requests.Response) -> Union[dict, bytes]:
try:
return result.json()
except ValueError:
return result.content
class WorkerClient:
def __init__(self, worker):
def __init__(self, url: str = None, secret: str = None, worker=None, name: str = None):
self._worker = worker
self._api = None
self._url = url or worker.url
self._secret = secret or worker.portal_secret
self._session = None
self._name = name
@property
def worker_name(self):
if self.worker:
return self.worker.log_name
if self._name:
return self._name
if self._url:
return self._url
@property
def _auth_header(self):
return {'Authorization': f'Bearer {self._secret}'}
@property
def worker_url(self):
return self._url
@property
def session(self) -> requests.Session:
if not self._session:
self._session = requests.session()
self._session.headers.update(self._auth_header)
return self._session
@property
def worker(self):
......@@ -22,16 +59,63 @@ class WorkerClient:
)
return cred
@property
def api(self) -> worker_api.client.WorkerClient:
if self._api is None:
self._api = worker_api.client.WorkerClient(**self.credentials)
return self._api
def make_request(self, path, method='get', **kwargs):
url = self._get_url(path)
log.debug(f"[W-REQ] ({method}) {url}")
result = self.session.request(url=url, method=method, **kwargs)
if not result.ok:
log.warning(f"[W-API] Worker returned the error "
f"({result.status_code}): {result.content}")
else:
log.debug(f"[W-RES] ({result.status_code}) {result.content}")
return _parse_result(result)
def execute_submission(self, submission):
log.info(f"[WC] Executing: {submission.log_name}")
log.info(f"[W-API] Executing in {self.worker_name}: {submission.log_name}")
parameters = submission.parameters
parameters['test_files_hash'] = submission.project.config.test_files_commit_hash
parameters['id'] = submission.id
result = self.api.submissions.create(parameters, sid=submission.id)
log.debug(f"[WC] Exec call result: {result}")
result = self.make_request(f'/submissions/{submission.id}', method='post', json=parameters)
return result
def get_submission(self, submission):
log.info(f"[W-API] Get submission in {self.worker_name}: {submission.log_name}")
result = self.make_request(f'/submissions/{submission.id}', method='get')
log.debug(f"[W-API] Submission: {result}")
return result
def list_submissions(self):
log.info(f"[W-API] Get submissions for {self.worker_name}")
result = self.make_request(f'/submissions', method='get')
log.debug(f"[W-API] Submission: {result}")
return result
def status(self):
log.info(f"[W-API] Get status for {self.worker_name}")
result = self.make_request(f'/management/status', method='get')
log.debug(f"[W-API] Status: {result}")
return result
def list_images(self, ):
log.info(f"[W-API] Get images for {self.worker_name}")
result = self.make_request(f'/submissions', method='get')
log.debug(f"[W-API] Images: {result}")
return result
def remove_image(self, project):
commit_hash = project.config.test_files_commit_hash
log.info(f"[W-API] Removing image from test files in {self.worker_name} for "
f"{project.log_name}: {commit_hash}")
result = self.make_request(f'/images/{commit_hash}/clean', method='get')
log.debug(f"[W-API] State: {result}")
return result
def register(self):
log.info(f"[W-API] Register worker {self.worker_name}")
result = self.make_request(f'/management/registration', method='post')
log.debug(f"[W-API] State: {result}")
return result
def _get_url(self, path: str):
return urllib.parse.urljoin(self.worker_url, '/api/v1.0', path)
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment