Verified Commit 2f5d1e5b authored by Peter Stanko's avatar Peter Stanko
Browse files

Fixing the images

parent 74897aae

kontr_worker/execution/containers.py

+33 −12
Original line number Diff line number Diff line
@@ -4,11 +4,13 @@ Containers module 
import logging

import os

from pathlib import Path

from typing import Optional



import docker

import flask

from docker.errors import APIError, ImageNotFound

from docker.models.containers import Container

from docker.models.networks import Network

from docker.models.images import Image



from kontr_worker.collections import TestFilesHashesCollection

from kontr_worker.entities import Submission, SubmissionDirCollection
@@ -75,15 +77,6 @@ class DockerWrapper: 
        """

        return Path(self.config['MOUNT_DIR'])



    @property

    def internal_docker_network(self) -> Network:

        """Gets internal docker network for the containers

        Returns(docker.models.networks.Network): Docker network instance



        """

        net_name = self.config['INTERNAL_DOCKER_NETWORK']

        return self.docker.networks.get(net_name)



    def get_instance(self, submission: Submission) -> 'Instance':

        """Creates instance of the custom docker container wrapper

        Args:
@@ -97,6 +90,31 @@ class DockerWrapper: 
    def get_container(self, submission: Submission) -> Container:

        return self.docker.containers.get(submission.container_id)



    def get_image(self, name) -> Optional[Image]:

        """Gets a docker image instance from the registry

        Args:

            name(str): Name of the image



        Returns(Optional[Image]): Docker image or None

        """

        try:

            return self.docker.images.get(name)

        except ImageNotFound as ex:

            log.warning(f"Docker image \"{name}\" not found in the registry: {ex}")

            return None



    def remove_image(self, name, **kwargs):

        try:

            self.docker.images.remove(name, **kwargs)

        except APIError as ex:

            log.error(f"[RMI] Cannot remove the image \"{name}\": {ex}")



    def build_image(self, **kwargs):

        try:

            self.docker.images.build(**kwargs)

        except APIError as ex:

            log.error(f"[RMI] Cannot build the image: {ex}")



    def stop(self, submission: Submission):

        """Stops the container

        Args:
@@ -139,7 +157,8 @@ class Instance(object): 


    def create(self, **kwargs) -> Container:

        # https://docker-py.readthedocs.io/en/stable/containers.html

        params = {**self._default_runtime_params, **kwargs}

        defaults = self._default_runtime_params or {}

        params = {**defaults, **kwargs}

        log.info(f"[DOCKER] Creating container with params: {params}")

        self.container = self.docker.containers.create(**params)

        return self.container
@@ -178,7 +197,9 @@ class Instance(object): 
        return self.__prepare_volume('result_files', 'rw')



    @property

    def _default_runtime_params(self) -> dict:

    def _default_runtime_params(self) -> Optional[dict]:

        if self.image_name is None:

            return None

        params = dict(

            image=self.image_name,

            name=f"kontr-submission-{self.submission.id}",

kontr_worker/execution/execution.py

+14 −6
Original line number Diff line number Diff line
@@ -97,11 +97,19 @@ class DockerExecutor(AbstractExecutor): 
        old_image_name = context.hashes.get(self.submission.unique_project_id)

        if old_image_name:

            log.info(f"[DOCKER] Deleting old image: {old_image_name}")

            context.docker.docker.images.remove(old_image_name)

            context.docker.remove_image(old_image_name)



    def process_test_files(self):

        test_files_hash = context.hashes.get(self.submission.unique_project_id)

        if test_files_hash is None or test_files_hash != self.submission.test_files_hash:



        if test_files_hash is not None and test_files_hash == self.submission.test_files_hash:

            log.debug("[SKIP] Processing not required - image already exists")

            if context.docker.get_image(test_files_hash) is not None:

                log.debug("[SKIP] Image exists in the registry")

                return

            else:

                log.warning("Image does not exists in the registry - needs a clean up")



        log.debug(f"Updating the image to [{self.submission.test_files_hash}]")

        self.checkout_test_files()

        self.clean_old_image()

kontr_worker/service/management.py

+1 −1
Original line number Diff line number Diff line
@@ -32,7 +32,7 @@ class ManagementService: 
        if context.hashes.get(test_files_hash):

            log.info(f"[DEL] Deleting old image: {test_files_hash}")

            context.hashes.remove(test_files_hash)

            context.docker.docker.images.remove(test_files_hash)

            context.docker.remove_image(test_files_hash)



    def clean_all_images(self):

        log.debug("[DEL] Deleting all images")

tests/exec/test_docker_wrapper.py

+3 −9
Original line number Diff line number Diff line 
from flask import Flask

import mock

from pathlib import Path



import pytest

from flask import Flask



from kontr_worker.entities import Submission

from kontr_worker.extensions import context
@@ -39,12 +39,6 @@ class NetworkStub(object): 
        return 'internal-network'





@mock.patch('docker.models.networks.NetworkCollection.get', NetworkStub)

def test_docker_wrapper_has_correct_network_name(app: Flask):

    network_name = app.config['INTERNAL_DOCKER_NETWORK']

    assert context.docker.internal_docker_network.name == network_name





def test_docker_wrapper_can_create_instance(app: Flask, test_submission: Submission):

    instance = context.docker.get_instance(test_submission)

    assert instance is not None