Verified Commit 2630998e authored by Peter Stanko's avatar Peter Stanko
Browse files

Logging has been extracted and parametrized to the class

parent cc6ff283

app.py

+0 −2
Original line number Diff line number Diff line
@@ -12,8 +12,6 @@ import portal 
from management.data import DataManagement

from portal import create_app, db, logger



logger.load_config()



log = logger.get_logger(__name__)



data_cli = AppGroup('data', help='Sample data initialization')

portal/__init__.py

+12 −5
Original line number Diff line number Diff line
@@ -8,7 +8,7 @@ from typing import Union 
from celery import Celery

from flask import Flask

from flask_cors import CORS

from flask_jwt_extended import JWTManager, get_jwt_identity

from flask_jwt_extended import JWTManager

from flask_migrate import Migrate

from flask_oauthlib.client import OAuth

from flask_sqlalchemy import SQLAlchemy
@@ -16,6 +16,7 @@ from storage import Storage 


from portal import logger, rest

from portal.config import CONFIGURATIONS

from portal.logger import Logging

from portal.tools.gitlab_client import GitlabFactory

from portal.tools.ldap_client import LDAPWrapper

from portal.tools.paths import ROOT_DIR
@@ -46,11 +47,15 @@ def configure_app(app: Flask, env: str = None, 
    app.config.from_object(config_object)

    _load_portal_local(app, env, ignore_local)

    app.config['PORTAL_ENV'] = config_type

    if app.config.get('PORTAL_LOG_CONFIG'):

    return app





def _log_config(app):

    if not app.config.get('PORTAL_LOG_CONFIG'):

        return

    log.trace("[INIT] Loaded config: ")

    for (key, val) in app.config.items():

        log.trace(f"[CONFIG] {key}={val}")

    return app





def _load_portal_local(app, env, ignore_local):
@@ -117,6 +122,8 @@ def create_app(environment: str = None): 
    app = Flask('portal')

    # app configuration

    configure_app(app, env=environment)

    Logging(app).load_config()

    _log_config(app)

    configure_storage(app)

    configure_extensions(app)

    configure_async(app)

portal/config.py

+10 −2
Original line number Diff line number Diff line
@@ -41,6 +41,9 @@ class Config(object): 
    GITLAB_BASE_DOMAIN = "gitlab.fi.muni.cz"

    GIT_REPO_BASE = os.getenv('GIT_REPO_BASE', f"git@{GITLAB_BASE_DOMAIN}")

    GITLAB_URL = os.getenv('GITLAB_URL', None)

    LOG_LEVEL_GLOBAL = os.getenv('LOG_LEVEL_GLOBAL', 'INFO')

    LOG_LEVEL_FILE = os.getenv('LOG_LEVEL_FILE', LOG_LEVEL_GLOBAL)

    LOG_LEVEL_CONSOLE = os.getenv('LOG_LEVEL_CONSOLE', LOG_LEVEL_GLOBAL)





class DevelopmentConfig(Config):
@@ -69,6 +72,9 @@ class ProductionConfig(Config): 
    """Production configuration

    """

    DEBUG = False

    LOG_LEVEL_GLOBAL = os.getenv('LOG_LEVEL_GLOBAL', 'INFO')

    LOG_LEVEL_FILE = os.getenv('LOG_LEVEL_FILE', LOG_LEVEL_GLOBAL)

    LOG_LEVEL_CONSOLE = os.getenv('LOG_LEVEL_CONSOLE', LOG_LEVEL_GLOBAL)





class TestConfig(Config):
@@ -96,7 +102,9 @@ class TestConfig(Config): 
    CELERY_RESULT_BACKEND = BROKER_URL

    PORTAL_LOG_CONFIG = False

    GIT_REPO_BASE = os.getenv('GIT_REPO_BASE', f"git@gitlab.local")



    LOG_LEVEL_GLOBAL = 'DEBUG'

    LOG_LEVEL_FILE = LOG_LEVEL_GLOBAL

    LOG_LEVEL_CONSOLE = LOG_LEVEL_GLOBAL



# pylint: enable=too-few-public-methods

portal/logger.py

+125 −77
Original line number Diff line number Diff line
@@ -5,71 +5,130 @@ Logging configuration module 
import logging

from logging.config import dictConfig



import flask



from portal.tools import paths





def get_logger_file(name):

class Logging:

    def __init__(self, app: flask.Flask = None):

        self._app = app



    @property

    def global_log_level(self):

        if not self._app:

            return 'INFO'

        return self._app.config.get('LOG_LEVEL_GLOBAL', 'INFO')



    @property

    def file_log_level(self):

        if not self._app:

            return 'INFO'

        return self._app.config.get('LOG_LEVEL_FILE', self.global_log_level)



    @property

    def console_log_level(self):

        if not self._app:

            return 'INFO'

        return self._app.config.get('LOG_LEVEL_CONSOLE', self.global_log_level)



    @property

    def handlers(self):

        return {

        'level': 'INFO',

        'class': 'logging.handlers.RotatingFileHandler',

        'formatter': 'verbose',

        'filename': str(paths.LOG_DIR / f'{name}.log'),

        'maxBytes': 5000000,  # 5MB

        'backupCount': 5

            'console': self.get_handler_console(),

            'portal_file': self.get_logger_file('portal'),

            'access_file': self.get_logger_file('access'),

            'auth_file': self.get_logger_file('auth'),

            'storage_file': self.get_logger_file('storage'),

            'flask_file': self.get_logger_file('flask')

        }



    @property

    def formatters(self):

        return {

            'verbose': {'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s'},



FORMATTERS = {

    'verbose': {

        'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s'

    },



    'simple': {

        'format': '%(levelname)s %(message)s'

    },

            'simple': {'format': '%(levelname)s %(message)s'},

            'colored_console': {

                '()': 'coloredlogs.ColoredFormatter',

        'format': "%(asctime)s - %(name)s - %(levelname)s - %(message)s", 'datefmt': '%H:%M:%S'

                'format': "%(asctime)s - %(name)s - %(levelname)s - %(message)s",

                'datefmt': '%H:%M:%S'

            },

        }



HANDLERS = {

    'console': {

        'level': 'DEBUG',

        'class': 'logging.StreamHandler',

        'formatter': 'colored_console'

    @property

    def loggers(self) -> dict:

        return {

            'portal': {

                'handlers': ['console', 'portal_file'],

                'level': self.global_log_level, 'propagate': True

            },

    'portal_file': get_logger_file('portal'),

    'access_file': get_logger_file('access'),

    'auth_file': get_logger_file('auth'),

    'storage_file': get_logger_file('storage'),

    'flask_file': get_logger_file('flask')

}



LOGGERS = {

    'portal': {'handlers': ['console', 'portal_file'], 'level': 'INFO', 'propagate': True},

            'portal.auth_log': {

        'handlers': ['console', 'auth_file'], 'level': 'INFO', 'propagate': True

                'handlers': ['console', 'auth_file'],

                'level': self.global_log_level, 'propagate': True

            },

            'portal.access_log': {

        'handlers': ['console', 'access_file'], 'level': 'DEBUG', 'propagate': True

                'handlers': ['console', 'access_file'],

                'level': self.global_log_level, 'propagate': True

            },

            'tests': {

                'handlers': ['console'],

                'level': self.global_log_level, 'propagate': True

            },

            'management': {

                'handlers': ['console'], 'level': self.global_log_level, 'propagate': True

            },

            'app': {

                'handlers': ['console'], 'level': self.global_log_level, 'propagate': True

            },

            'flask': {

                'handlers': ['console', 'flask_file'], 'level': self.global_log_level,

                'propagate': True

            },

            'werkzeug': {

                'handlers': ['console'], 'level': self.global_log_level, 'propagate': True

            },

            'storage': {

                'handlers': ['console', 'storage_file'], 'level': self.global_log_level,

                'propagate': True

            },

    'tests': {'handlers': ['console'], 'level': 'DEBUG', 'propagate': True},

    'management': {'handlers': ['console'], 'level': 'INFO', 'propagate': True},

    'app': {'handlers': ['console'], 'level': 'DEBUG', 'propagate': True},

    'flask': {'handlers': ['console', 'flask_file'], 'level': 'INFO', 'propagate': True},

    'werkzeug': {'handlers': ['console'], 'level': 'INFO', 'propagate': True},

    'storage': {'handlers': ['console', 'storage_file'], 'level': 'INFO', 'propagate': True},

        }



LOGGING_CONF = {

    @property

    def config(self):

        return {

            'version': 1,

            'disable_existing_loggers': True,

    'formatters': FORMATTERS,

    'handlers': HANDLERS,

    'loggers': LOGGERS,

            'formatters': self.formatters,

            'handlers': self.handlers,

            'loggers': self.loggers,

        }



    def get_logger_file(self, name, level: str = None):

        level = level or self.file_log_level

        return {

            'level': level,

            'class': 'logging.handlers.RotatingFileHandler',

            'formatter': 'verbose',

            'filename': str(paths.LOG_DIR / f'{name}.log'),

            'maxBytes': 5000000,  # 5MB

            'backupCount': 5

        }



    def get_handler_console(self, level: str = None):

        level = level or self.console_log_level

        return {

            'level': level, 'class': 'logging.StreamHandler', 'formatter': 'colored_console'

        }



    def load_config(self):

        """Loads config based on the config type

        Args:

        """

        add_custom_log_level()

        dictConfig(self.config)





TRACE_LOG_LVL = 9
@@ -84,29 +143,18 @@ def add_custom_log_level(): 
    logging.Logger.trace = _trace





def load_config(conf_type=None):

    """Loads config based on the config type

    Args:

        conf_type(str): Config type available (dev, test, prod)



    """

    if conf_type == 'test':

        LOGGING_CONF['loggers']['management']['level'] = 'WARNING'

    add_custom_log_level()

    dictConfig(LOGGING_CONF)





def get_logger(*args, **kwargs):

    logger = logging.getLogger(*args, **kwargs)

    return logger





def get_access_logger(*args, **kwargs):

    return logging.getLogger('portal.access_log', *args, **kwargs)

def get_access_logger():

    return logging.getLogger('portal.access_log')





def get_auth_logger():

    return logging.getLogger('portal.auth_log')



def get_auth_logger(*args, **kwargs):

    return logging.getLogger('portal.auth_log', *args, **kwargs)



ACCESS = get_access_logger()

AUTH = get_auth_logger()

tests/conftest.py

+1 −3
Original line number Diff line number Diff line
@@ -5,11 +5,9 @@ import pytest 
from management.data import DataManagement

from portal import create_app, db

from portal.database import ProjectConfig

from portal.logger import load_config

from portal.logger import Logging

from tests.utils.ent_mocker import EntitiesMocker



load_config('test')





@pytest.fixture(scope='function')

def app():