Verified Commit a9c5c378 authored by Peter Stanko's avatar Peter Stanko
Browse files

Worker API has been removed

parent 4525b390
......@@ -25,7 +25,6 @@ celery = {version = "*", extras = ["auth", "redis", "yaml", "msgpack"]}
mockredispy = "*"
python-slugify = "*"
coloredlogs = "*"
worker-api = {editable = true, git = "https://gitlab.fi.muni.cz/grp-kontr2/worker-api.git", ref='master'}
"ldap3" = "*"
[dev-packages]
......
{
"_meta": {
"hash": {
"sha256": "78ad58bc50300482ca022013499e01eb3facfde2657d9913a6060f2388cf9bf0"
"sha256": "f94131fd9b9fb9983abd7a9804e6f01746cb6c56a644a15e4310ba395e0e868d"
},
"pipfile-spec": 6,
"requires": {
......@@ -352,11 +352,11 @@
},
"marshmallow": {
"hashes": [
"sha256:276db2f676763649262cd957757298329b07a20be513c1880e2763a0523260bf",
"sha256:485ac6ed0dff5e1af6ea1e3a54425a448968f581b065424c89a5375e4d4866fd"
"sha256:24a76fdc8857344fe896faa99b589ce10f44c691ce511fee37775757d482f036",
"sha256:ec7a8d0101c0175480684f8b6431ec52dc0b3e05bbd780643c2ee7df7735f148"
],
"index": "pypi",
"version": "==2.15.6"
"version": "==2.16.0"
},
"marshmallow-enum": {
"hashes": [
......@@ -582,11 +582,6 @@
"sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b"
],
"version": "==0.14.1"
},
"worker-api": {
"editable": true,
"git": "https://gitlab.fi.muni.cz/grp-kontr2/worker-api.git",
"ref": "8b24f7c0290c6b3447692f081fb67a1df6868b73"
}
},
"develop": {
......@@ -721,11 +716,11 @@
},
"py": {
"hashes": [
"sha256:06a30435d058473046be836d3fc4f27167fd84c45b99704f2fb5509ef61f9af1",
"sha256:50402e9d1c9005d759426988a492e0edaadb7f4e68bcddfea586bc7432d009c6"
"sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694",
"sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6"
],
"markers": "python_version != '3.1.*' and python_version != '3.3.*' and python_version >= '2.7' and python_version != '3.2.*' and python_version != '3.0.*'",
"version": "==1.6.0"
"markers": "python_version != '3.0.*' and python_version >= '2.7' and python_version != '3.1.*' and python_version != '3.3.*' and python_version != '3.2.*'",
"version": "==1.7.0"
},
"pylint": {
"hashes": [
......
......@@ -132,7 +132,7 @@ class SubmissionProcessor:
return worker
def execute_submission(self, worker: Worker):
worker_client = tools.worker_client.WorkerClient(worker=worker)
worker_client = self._rest.workers(worker).worker_client
worker_client.execute_submission(self.submission)
def _worker_not_available(self):
......
"""
Components service
"""
import portal.tools.worker_client
from portal import logger
from portal.database.models import Worker
......@@ -18,6 +19,12 @@ class WorkerService(GeneralService):
def worker(self):
return self._entity
@property
def worker_client(self) -> portal.tools.worker_client.WorkerClient:
if not hasattr(self, '__worker_client'):
setattr(self, '__worker_client', portal.tools.worker_client.WorkerClient(self._entity))
return getattr(self, '__worker_client')
def _set_data(self, entity: Worker, **data) -> Worker:
allowed = ['name', 'url', 'tags', 'portal_secret', 'state']
return self.update_entity(entity, data, allowed=allowed)
......
import worker_api.client
import urllib.parse
from typing import Union
import requests
from portal import logger
log = logger.get_logger(__name__)
def _parse_result(result: requests.Response) -> Union[dict, bytes]:
try:
return result.json()
except ValueError:
return result.content
class WorkerClient:
def __init__(self, worker):
def __init__(self, url: str = None, secret: str = None, worker=None, name: str = None):
self._worker = worker
self._api = None
self._url = url or worker.url
self._secret = secret or worker.portal_secret
self._session = None
self._name = name
@property
def worker_name(self):
if self.worker:
return self.worker.log_name
if self._name:
return self._name
if self._url:
return self._url
@property
def _auth_header(self):
return {'Authorization': f'Bearer {self._secret}'}
@property
def worker_url(self):
return self._url
@property
def session(self) -> requests.Session:
if not self._session:
self._session = requests.session()
self._session.headers.update(self._auth_header)
return self._session
@property
def worker(self):
......@@ -22,16 +59,63 @@ class WorkerClient:
)
return cred
@property
def api(self) -> worker_api.client.WorkerClient:
if self._api is None:
self._api = worker_api.client.WorkerClient(**self.credentials)
return self._api
def make_request(self, path, method='get', **kwargs):
url = self._get_url(path)
log.debug(f"[W-REQ] ({method}) {url}")
result = self.session.request(url=url, method=method, **kwargs)
if not result.ok:
log.warning(f"[W-API] Worker returned the error "
f"({result.status_code}): {result.content}")
else:
log.debug(f"[W-RES] ({result.status_code}) {result.content}")
return _parse_result(result)
def execute_submission(self, submission):
log.info(f"[WC] Executing: {submission.log_name}")
log.info(f"[W-API] Executing in {self.worker_name}: {submission.log_name}")
parameters = submission.parameters
parameters['test_files_hash'] = submission.project.config.test_files_commit_hash
parameters['id'] = submission.id
result = self.api.submissions.create(parameters, sid=submission.id)
log.debug(f"[WC] Exec call result: {result}")
result = self.make_request(f'/submissions/{submission.id}', method='post', json=parameters)
return result
def get_submission(self, submission):
log.info(f"[W-API] Get submission in {self.worker_name}: {submission.log_name}")
result = self.make_request(f'/submissions/{submission.id}', method='get')
log.debug(f"[W-API] Submission: {result}")
return result
def list_submissions(self):
log.info(f"[W-API] Get submissions for {self.worker_name}")
result = self.make_request(f'/submissions', method='get')
log.debug(f"[W-API] Submission: {result}")
return result
def status(self):
log.info(f"[W-API] Get status for {self.worker_name}")
result = self.make_request(f'/management/status', method='get')
log.debug(f"[W-API] Status: {result}")
return result
def list_images(self, ):
log.info(f"[W-API] Get images for {self.worker_name}")
result = self.make_request(f'/submissions', method='get')
log.debug(f"[W-API] Images: {result}")
return result
def remove_image(self, project):
commit_hash = project.config.test_files_commit_hash
log.info(f"[W-API] Removing image from test files in {self.worker_name} for "
f"{project.log_name}: {commit_hash}")
result = self.make_request(f'/images/{commit_hash}/clean', method='get')
log.debug(f"[W-API] State: {result}")
return result
def register(self):
log.info(f"[W-API] Register worker {self.worker_name}")
result = self.make_request(f'/management/registration', method='post')
log.debug(f"[W-API] State: {result}")
return result
def _get_url(self, path: str):
return urllib.parse.urljoin(self.worker_url, '/api/v1.0', path)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment