Verified Commit 0208f3d8 authored by Peter Stanko's avatar Peter Stanko
Browse files

Fixed logging and logging loading

parent 273fc1c0
portal.local.cfg
.git
.git/
Dockerfile
.gitignore
.log/*
.idea/
......@@ -3,6 +3,9 @@ Main application module
- Creates instance of the flask app named app
- registers commands
"""
from portal.logger import Logging
Logging().load_config()
import click
from flask import Flask
......@@ -20,6 +23,7 @@ courses_cli = AppGroup('courses', help='Courses management')
devel_cli = AppGroup('devel', help='Development management')
submissions_cli = AppGroup('submissions', help='Submissions management')
projects_cli = AppGroup('projects', help='Projects management')
cfg_cli = AppGroup('cfg', help='Configuration management')
app: Flask = create_app()
......@@ -35,7 +39,15 @@ manager = DataManagement(app, db)
celery = portal.get_celery(app)
@devel_cli.command('run', help='Runs the devel server with initializes db')
@cfg_cli.command('env', help='Print flask configuration')
@click.pass_context
def cli_print_cfg(ctx):
log.info("[CFG] Configuration:")
for (key, val) in app.config.items():
print(f"{key}: {val}")
@devel_cli.command('run', help='Runs the devel server with iniializes db')
@click.option('-p', '--port', default=8000)
@click.pass_context
def cli_run_devel(ctx, port):
......
......@@ -5,18 +5,17 @@ Main Portal module
import os
from typing import Union
from authlib.flask.client import OAuth
from celery import Celery
from flask import Flask
from flask_cors import CORS
from flask_jwt_extended import JWTManager
from flask_migrate import Migrate
from authlib.flask.client import OAuth
from flask_sqlalchemy import SQLAlchemy
from portal.storage import Storage
from portal import logger, rest
from portal.config import CONFIGURATIONS
from portal.logger import Logging
from portal.storage import Storage
from portal.tools.gitlab_client import GitlabFactory
from portal.tools.is_api_adapter import IsApiFactory
from portal.tools.ldap_client import LDAPWrapper
......@@ -123,7 +122,6 @@ def create_app(environment: str = None):
app = Flask('portal')
# app configuration
configure_app(app, env=environment)
Logging(app).load_config()
_log_config(app)
configure_storage(app)
configure_extensions(app)
......
......@@ -105,6 +105,7 @@ class SubmissionProcessor:
except Exception as ex:
log.error(f"[PROC] Storage submission download aborted due error: {ex}")
self.abort_submission('Unable to download a submission')
return None
self.dispatch_submission_processing()
def revoke_task(self):
......
......@@ -3,34 +3,29 @@ Logging configuration module
"""
import logging
import os
from logging.config import dictConfig
import flask
from portal.tools import paths
class Logging:
def __init__(self, app: flask.Flask = None):
self._app = app
def __init__(self, global_level: str = None, file_level=None, console_level=None):
self._global_level = global_level or os.getenv('LOG_LEVEL_GLOBAL')
self._file_level = file_level or os.getenv('LOG_LEVEL_FILE', self._global_level)
self._console_level = console_level or os.getenv('LOG_LEVEL_CONSOLE', self._global_level)
@property
def global_log_level(self):
if not self._app:
return 'INFO'
return self._app.config.get('LOG_LEVEL_GLOBAL', 'INFO')
return self._global_level
@property
def file_log_level(self):
if not self._app:
return 'INFO'
return self._app.config.get('LOG_LEVEL_FILE', self.global_log_level)
return self._file_level
@property
def console_log_level(self):
if not self._app:
return 'INFO'
return self._app.config.get('LOG_LEVEL_CONSOLE', self.global_log_level)
return self._console_level
@property
def handlers(self):
......@@ -40,7 +35,6 @@ class Logging:
'flask_file': self.get_logger_file('flask'),
'access_file': self.get_logger_file('access'),
'auth_file': self.get_logger_file('auth'),
'storage_file': self.get_logger_file('storage'),
'login_file': self.get_logger_file('login'),
'submit_file': self.get_logger_file('submit'),
'files_access_file': self.get_logger_file('files_access'),
......@@ -103,20 +97,21 @@ class Logging:
'werkzeug': {
'handlers': ['console'], 'level': self.global_log_level, 'propagate': True
},
'portal.storage': {
'handlers': ['console', 'storage_file'], 'level': self.global_log_level,
'propagate': True
'git': {
'handlers': ['console'], 'level': self.global_log_level, 'propagate': True
},
}
@property
def config(self):
loggers = self.loggers
return {
'version': 1,
'disable_existing_loggers': True,
'formatters': self.formatters,
'handlers': self.handlers,
'loggers': self.loggers,
'loggers': loggers,
}
def get_logger_file(self, name, level: str = None):
......@@ -133,7 +128,9 @@ class Logging:
def get_handler_console(self, level: str = None):
level = level or self.console_log_level
return {
'level': level, 'class': 'logging.StreamHandler', 'formatter': 'colored_console'
'level': level,
'class': 'logging.StreamHandler',
'formatter': 'colored_console'
}
def load_config(self):
......@@ -142,6 +139,12 @@ class Logging:
"""
add_custom_log_level()
dictConfig(self.config)
self.reenable_loggers()
def reenable_loggers(self):
loggers = [logging.getLogger(name) for name in logging.root.manager.loggerDict]
for logger in loggers:
logger.disabled = False
TRACE_LOG_LVL = 9
......
......@@ -4,6 +4,7 @@ from pathlib import Path
from git import Repo
log = logging.getLogger(__name__)
log.disabled = False
def git_clone(entity, **kwargs):
......@@ -74,7 +75,7 @@ class GitWrapper:
Returns:
"""
log.info(f"[GIT] Cloning from: {self.config.url} -> {self.entity.workspace}")
log.info(f"[GIT] Cloning from {self.config.url} -> {self.entity.workspace}")
params = dict(
url=self.config.url,
to_path=self.entity.workspace
......@@ -84,7 +85,7 @@ class GitWrapper:
if self._custom_ssh_key:
params['env'] = dict(GIT_SSH_COMMAND=self._create_ssh_command())
log.debug(f"[GIT] Clone: {params}")
cloned_repo = Repo.clone_from(**params)
self._process_cloned_repo(cloned_repo)
self._repo = cloned_repo
......@@ -100,4 +101,7 @@ class GitWrapper:
def _create_ssh_command(self):
ssh_key_path = str(Path(self._custom_ssh_key).absolute())
return f"ssh -i {ssh_key_path}"
log.debug(f"[GIT] Using custom ssh key: {ssh_key_path}")
ssh_cmd = f"ssh -i {ssh_key_path}"
log.debug(f"[GIT] Using custom ssh command: {ssh_cmd}")
return ssh_cmd
from portal.logger import Logging
Logging(global_level='DEBUG').load_config()
import os
import pytest
......
......@@ -64,9 +64,6 @@ def test_list_all_avail_for_user_as_teacher(rest_service, client, teacher1_crede
assert_response(response, 200)
resp_submissions = rest_tools.extract_data(response)
for sub in resp_submissions:
subm = rest_service.find.submission(sub['id'])
print([role.log_name for role in rest_service.find.user('teacher1').roles])
assert len(resp_submissions) == 5
......
......@@ -293,9 +293,7 @@ def test_user_reviews(client):
user_reviews = rest_tools.extract_data(response)
assert len(user_reviews) == len(reviews)
# the test is incomplete - only print
print(f"user_reviews: {user_reviews}")
def test_user_projects(client):
user = User.query.filter_by(username="student2").first()
......
import logging.config
import logging
import os
from pathlib import Path
import pytest
from portal import create_app
from portal.storage import Storage
TESTS_PATH = Path(__file__).parent
RESOURCES_PATH = TESTS_PATH / 'resources'
@pytest.fixture(scope='session')
def app():
os.environ["PORTAL_CONFIG_TYPE"] = 'test'
_app = create_app(environment='test')
with _app.app_context():
yield _app
@pytest.fixture
def log():
return logging.getLogger(__name__)
......@@ -35,7 +45,8 @@ def results_dir(tmpdir):
@pytest.fixture
def storage_instance(test_files_dir, workspace_dir, submissions_dir, results_dir) -> Storage:
def storage_instance(app, test_files_dir, workspace_dir,
submissions_dir, results_dir) -> Storage:
return Storage(test_files_dir=test_files_dir,
workspace_dir=workspace_dir,
submissions_dir=submissions_dir,
......
......@@ -36,7 +36,7 @@ def uploaded_entity(submission, workspace, git_config):
@pytest.fixture
def subm_proc(uploaded_entity):
def subm_proc(uploaded_entity, app):
return GitWrapper(uploaded_entity)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment