Verified Commit 56534c03 authored by Peter Stanko's avatar Peter Stanko
Browse files

Access logging and accountability support

parent 47aa4200
Pipeline #16575 passed with stage
in 8 minutes and 41 seconds
......@@ -8,7 +8,7 @@ from typing import Union
from celery import Celery
from flask import Flask
from flask_cors import CORS
from flask_jwt_extended import JWTManager
from flask_jwt_extended import JWTManager, get_jwt_identity
from flask_migrate import Migrate
from flask_oauthlib.client import OAuth
from flask_sqlalchemy import SQLAlchemy
......@@ -127,7 +127,6 @@ def create_app(environment: str = None):
from flask import request
body = request.get_data(as_text=True) if request.data else ''
app.logger.debug(f"[{request.method}] {request.url} - {body}")
return app
......
......@@ -7,6 +7,18 @@ from logging.config import dictConfig
from portal.tools import paths
def get_logger_file(name):
return {
'level': 'DEBUG',
'class': 'logging.handlers.RotatingFileHandler',
'formatter': 'verbose',
'filename': str(paths.LOG_DIR / f'{name}.log'),
'maxBytes': 5000000, # 5MB
'backupCount': 5
}
FORMATTERS = {
'verbose': {
'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
......@@ -27,24 +39,23 @@ HANDLERS = {
'class': 'logging.StreamHandler',
'formatter': 'colored_console'
},
'file': {
'level': 'DEBUG',
'class': 'logging.handlers.RotatingFileHandler',
'formatter': 'verbose',
'filename': str(paths.LOG_DIR / 'portal.log'),
'maxBytes': 500000,
'backupCount': 5
}
'portal_file': get_logger_file('portal'),
'access_file': get_logger_file('access'),
'storage_file': get_logger_file('storage'),
'flask_file': get_logger_file('flask')
}
LOGGERS = {
'portal': {'handlers': ['console', 'file'], 'level': 'DEBUG', 'propagate': True},
'portal': {'handlers': ['console', 'portal_file'], 'level': 'DEBUG', 'propagate': True},
'portal.access_log': {
'handlers': ['console', 'access_file'], 'level': 'DEBUG', 'propagate': True
},
'tests': {'handlers': ['console'], 'level': 'DEBUG', 'propagate': True},
'management': {'handlers': ['console'], 'level': 'INFO', 'propagate': True},
'app': {'handlers': ['console'], 'level': 'DEBUG', 'propagate': True},
'flask': {'handlers': ['console'], 'level': 'DEBUG', 'propagate': True},
'flask': {'handlers': ['console', 'flask_file'], 'level': 'DEBUG', 'propagate': True},
'werkzeug': {'handlers': ['console'], 'level': 'DEBUG', 'propagate': True},
'storage': {'handlers': ['console'], 'level': 'INFO', 'propagate': True},
'storage': {'handlers': ['console', 'storage_file'], 'level': 'INFO', 'propagate': True},
}
LOGGING_CONF = {
......@@ -84,3 +95,10 @@ def load_config(conf_type=None):
def get_logger(*args, **kwargs):
logger = logging.getLogger(*args, **kwargs)
return logger
def get_access_logger(*args, **kwargs):
return logging.getLogger('portal.access_log', *args, **kwargs)
ACCESS = get_access_logger()
......@@ -6,6 +6,7 @@ from portal.database.models import ClientType
from portal.rest import rest_helpers
from portal.rest.custom_resource import CustomResource
from portal.rest.schemas import SCHEMAS
from portal.tools.decorators import access_log
client_namespace = Namespace('client')
clients_namespace = Namespace('clients')
......@@ -49,6 +50,7 @@ class ClientSecretsController(CustomResource):
return SCHEMAS.dump('secrets', client.secrets)
@jwt_required
@access_log
# @workers_namespace.response(201, 'Created worker secret', model=secret_schema)
def post(self, cid: str):
self.permissions.require.sysadmin_or_self(cid)
......@@ -72,6 +74,7 @@ class ClientSecretController(CustomResource):
return SCHEMAS.dump('secret', secret)
@jwt_required
@access_log
@clients_namespace.response(204, 'Client secret deleted')
def delete(self, cid: str, sid: str):
self.rest.permissions.require.sysadmin_or_self(cid)
......@@ -81,6 +84,7 @@ class ClientSecretController(CustomResource):
return '', 204
@jwt_required
@access_log
@clients_namespace.response(204, 'Client secret updated')
@clients_namespace.response(403, 'Not allowed to update client secret')
def put(self, cid: str, sid: str):
......
......@@ -9,6 +9,7 @@ from portal.rest.custom_resource import CustomResource
from portal.rest.schemas import SCHEMAS
from portal.service.errors import ForbiddenError, PortalAPIError
from portal.service.filters import filter_course_dump
from portal.tools.decorators import access_log
courses_namespace = Namespace('courses')
log = logger.get_logger(__name__)
......@@ -29,6 +30,7 @@ class CourseList(CustomResource):
@jwt_required
# @courses_namespace.response(200, 'Created course', model=course_schema)
# @courses_namespace.response(403, 'Not allowed to create course', model=course_schema)
@access_log
def post(self):
self.permissions.require.sysadmin()
......@@ -62,6 +64,7 @@ class CourseResource(CustomResource):
raise ForbiddenError(client=client)
@jwt_required
@access_log
@courses_namespace.response(204, 'Course deleted')
@courses_namespace.response(403, 'Not allowed to delete course')
def delete(self, cid: str):
......@@ -71,6 +74,7 @@ class CourseResource(CustomResource):
return '', 204
@jwt_required
@access_log
@courses_namespace.response(204, 'Course updated')
@courses_namespace.response(403, 'Not allowed to update course')
def put(self, cid: str):
......@@ -98,6 +102,7 @@ class CourseNotesToken(CustomResource):
return course.notes_access_token
@jwt_required
@access_log
@courses_namespace.response(204, 'Course\'s notes access token updated')
@courses_namespace.response(
403, 'Not allowed to update course\'s notes access token')
......@@ -117,6 +122,7 @@ class CourseNotesToken(CustomResource):
@courses_namespace.response(404, 'Course not found')
class CourseImport(CustomResource):
@jwt_required
@access_log
# @courses_namespace.response(200, 'Course has been imported', model=course_schema)
@courses_namespace.response(403, 'Not allowed to import course')
@courses_namespace.response(400, 'Cannot import course to itself.')
......
......@@ -6,6 +6,7 @@ from portal import logger
from portal.rest import rest_helpers
from portal.rest.custom_resource import CustomResource
from portal.rest.schemas import SCHEMAS
from portal.tools.decorators import access_log
groups_namespace = Namespace('')
......@@ -24,6 +25,7 @@ class GroupsList(CustomResource):
return SCHEMAS.dump('groups', groups)
@jwt_required
@access_log
# @groups_namespace.response(201, 'Group created', model=group_schema)
@groups_namespace.response(403, 'Not allowed to create group')
def post(self, cid: str):
......@@ -55,6 +57,7 @@ class GroupResource(CustomResource):
return SCHEMAS.dump('group', group)
@jwt_required
@access_log
@groups_namespace.response(204, 'Group deleted')
@groups_namespace.response(403, 'Not allowed to delete group')
def delete(self, cid: str, gid: str):
......@@ -67,6 +70,7 @@ class GroupResource(CustomResource):
return '', 204
@jwt_required
@access_log
@groups_namespace.response(204, 'Group updated')
@groups_namespace.response(403, 'Not allowed to update group')
def put(self, cid: str, gid: str):
......@@ -99,6 +103,7 @@ class GroupUsersList(CustomResource):
return SCHEMAS.dump('users', users)
@jwt_required
@access_log
@groups_namespace.response(204, 'User\'s list updated')
@groups_namespace.response(403, 'Not allowed to add users to the group')
def put(self, cid: str, gid: str):
......@@ -127,6 +132,7 @@ class GroupUsersList(CustomResource):
@groups_namespace.response(404, 'User not found')
class GroupAddOrDeleteSingleUser(CustomResource):
@jwt_required
@access_log
@groups_namespace.response(204, 'User\'s list updated')
@groups_namespace.response(403, 'Not allowed to add user to the group')
def put(self, cid: str, gid: str, uid: str):
......@@ -141,6 +147,7 @@ class GroupAddOrDeleteSingleUser(CustomResource):
return '', 204
@jwt_required
@access_log
@groups_namespace.response(204, 'User\'s list updated')
@groups_namespace.response(403, 'Not allowed to delete users to the group')
def delete(self, cid: str, gid: str, uid: str):
......@@ -181,6 +188,7 @@ class GroupProjectsList(CustomResource):
@groups_namespace.response(404, 'Project not found')
class GroupAddOrDeleteProject(CustomResource):
@jwt_required
@access_log
@groups_namespace.response(204, 'Projects\'s list updated')
@groups_namespace.response(403, 'Not allowed to add project to the group')
def put(self, cid: str, gid: str, pid: str):
......@@ -195,6 +203,7 @@ class GroupAddOrDeleteProject(CustomResource):
return '', 204
@jwt_required
@access_log
@groups_namespace.response(204, 'Projects\'s list updated')
@groups_namespace.response(
403, 'Not allowed to delete project from the group')
......@@ -215,6 +224,7 @@ class GroupAddOrDeleteProject(CustomResource):
@groups_namespace.response(404, 'Course not found')
class GroupImport(CustomResource):
@jwt_required
@access_log
@groups_namespace.response(201, 'Group import')
@groups_namespace.response(404, 'Group not found')
@groups_namespace.response(403, 'Not allowed to import to the group')
......
......@@ -8,6 +8,7 @@ from portal.rest import rest_helpers
from portal.rest.custom_resource import CustomResource
from portal.rest.schemas import SCHEMAS
from portal.service import errors
from portal.tools.decorators import access_log
projects_namespace = Namespace('') # pylint: disable=invalid-name
......@@ -27,6 +28,7 @@ class ProjectsList(CustomResource):
return SCHEMAS.dump('projects', projects)
@jwt_required
@access_log
@projects_namespace.response(404, 'Course not found')
# @projects_namespace.response(201, 'Project created', model=project_schema)
def post(self, cid: str):
......@@ -58,6 +60,7 @@ class ProjectResource(CustomResource):
return SCHEMAS.dump('project', project)
@jwt_required
@access_log
@projects_namespace.response(204, 'Project deleted')
@projects_namespace.response(403, 'Not allowed to delete project')
def delete(self, cid: str, pid: str):
......@@ -70,6 +73,7 @@ class ProjectResource(CustomResource):
return '', 204
@jwt_required
@access_log
@projects_namespace.response(204, 'Project updated')
@projects_namespace.response(403, 'Not allowed to update project')
def put(self, cid: str, pid: str):
......@@ -110,6 +114,7 @@ class ProjectConfigResource(CustomResource):
raise errors.ForbiddenError(perm_service.client)
@jwt_required
@access_log
@projects_namespace.response(204, 'Project config updated')
def put(self, cid: str, pid: str):
course = self.find.course(cid)
......@@ -129,6 +134,7 @@ class ProjectConfigResource(CustomResource):
@projects_namespace.response(404, 'Project not found')
class ProjectTestFilesRefresh(CustomResource):
@jwt_required
@access_log
@projects_namespace.response(204, 'Project test_files updated')
def post(self, cid: str, pid: str):
course = self.find.course(cid)
......@@ -160,6 +166,7 @@ class ProjectSubmissions(CustomResource):
return SCHEMAS.dump('submissions', submissions)
@jwt_required
@access_log
# @projects_namespace.response(201, 'Project submission create', model=submission_schema)
def post(self, cid: str, pid: str):
client = self.rest.auth.client
......
......@@ -7,6 +7,7 @@ from portal.database.models import ClientType
from portal.rest import rest_helpers
from portal.rest.custom_resource import CustomResource
from portal.rest.schemas import SCHEMAS
from portal.tools.decorators import access_log
roles_namespace = Namespace('')
......@@ -24,6 +25,7 @@ class RoleList(CustomResource):
return SCHEMAS.dump('roles', roles)
@jwt_required
@access_log
# @roles_namespace.response(201, 'Role created', model=role_schema)
def post(self, cid):
course = self.find.course(cid)
......@@ -52,6 +54,7 @@ class RoleResource(CustomResource):
return SCHEMAS.dump('role', role)
@jwt_required
@access_log
@roles_namespace.response(204, 'Deleted role')
def delete(self, cid, rid):
course = self.find.course(cid)
......@@ -63,6 +66,7 @@ class RoleResource(CustomResource):
return '', 204
@jwt_required
@access_log
@roles_namespace.response(204, 'Role updated')
def put(self, cid: str, rid: str):
course = self.find.course(cid)
......@@ -94,6 +98,7 @@ class RolePermissions(CustomResource):
return SCHEMAS.dump('permissions', role.permissions)
@jwt_required
@access_log
# @roles_namespace.response(204, 'Update permissions for course', model=permissions_schema)
def put(self, cid: str, rid: str):
course = self.find.course(cid)
......@@ -127,6 +132,7 @@ class RoleUsersList(CustomResource):
return SCHEMAS.dump('users', users)
@jwt_required
@access_log
@roles_namespace.response(204, 'Updates users membership in role')
def put(self, cid: str, rid: str):
course = self.find.course(cid)
......@@ -154,6 +160,7 @@ class RoleUsersList(CustomResource):
@roles_namespace.response(404, 'Client not found')
class RoleClient(CustomResource):
@jwt_required
@access_log
@roles_namespace.response(204, 'Adds permissions to role')
def put(self, cid: str, rid: str, clid: str):
course = self.find.course(cid)
......@@ -166,6 +173,7 @@ class RoleClient(CustomResource):
return '', 204
@jwt_required
@access_log
@roles_namespace.response(204, 'Removes permissions from role')
def delete(self, cid: str, rid: str, clid: str):
course = self.find.course(cid)
......
......@@ -5,6 +5,7 @@ from portal import logger, storage
from portal.rest import rest_helpers
from portal.rest.custom_resource import CustomResource
from portal.rest.schemas import SCHEMAS
from portal.tools.decorators import access_log
submissions_namespace = Namespace('submissions')
......@@ -33,6 +34,7 @@ class SubmissionResource(CustomResource):
return SCHEMAS.dump('submission', submission)
@jwt_required
@access_log
@submissions_namespace.response(204, 'Submission deleted')
def delete(self, sid: str):
self.permissions.require.sysadmin()
......@@ -54,6 +56,7 @@ class SubmissionState(CustomResource):
return SCHEMAS.dump('submission', submission)
@jwt_required
@access_log
@submissions_namespace.response(204, 'Submission state updated')
def put(self, sid: str):
client = self.rest.auth.client
......@@ -169,6 +172,7 @@ class SubmissionResultFiles(CustomResource):
return service.send_file_or_zip(storage_entity)
@jwt_required
@access_log
def post(self, sid: str):
submission = self.find.submission(sid)
# authorization
......@@ -185,6 +189,7 @@ class SubmissionResultFiles(CustomResource):
@submissions_namespace.response(404, 'Submissions not found')
class SubmissionResubmit(CustomResource):
@jwt_required
@access_log
# @submissions_namespace.response(201, 'New submission', model=submission_schema)
def post(self, sid: str):
source_submission = self.find.submission(sid)
......@@ -207,6 +212,7 @@ class SubmissionResubmit(CustomResource):
@submissions_namespace.response(204, 'Submissions canceled')
class SubmissionCancel(CustomResource):
@jwt_required
@access_log
# @submissions_namespace.response(201, 'New submission', model=submission_schema)
def post(self, sid: str):
submission = self.find.submission(sid)
......@@ -234,6 +240,7 @@ class SubmissionReview(CustomResource):
return SCHEMAS.dump('review', submission.review)
@jwt_required
@access_log
# @submissions_namespace.response(201, 'Submissions review created', model=review_schema)
def post(self, sid: str):
client = self.rest.auth.client
......
......@@ -13,6 +13,7 @@ from portal.rest.custom_resource import CustomResource
from portal.rest.schemas import SCHEMAS
from portal.service import errors
from portal.service.rest import RestService
from portal.tools.decorators import access_log
users_namespace = Namespace('users')
......@@ -37,6 +38,7 @@ class UserList(CustomResource):
return SCHEMAS.dump('users', filtered_users), 200
@jwt_required
@access_log
# @users_namespace.response(201, 'Created user', model=user_schema)
def post(self):
self.permissions().require.sysadmin()
......@@ -64,6 +66,7 @@ class UserResource(CustomResource):
return SCHEMAS.user_reduced()
@jwt_required
@access_log
@users_namespace.response(204, 'User updated')
def put(self, uid: str):
user = self.find.user(uid)
......@@ -75,6 +78,7 @@ class UserResource(CustomResource):
return '', 204
@jwt_required
@access_log
@users_namespace.response(204, 'User deleted')
def delete(self, uid: str):
self.permissions().require.sysadmin()
......@@ -88,6 +92,7 @@ class UserResource(CustomResource):
@users_namespace.response(404, 'User not found')
class UserPassword(CustomResource):
@jwt_required
@access_log
@users_namespace.response(204, 'User password updated')
def put(self, uid: str):
user = self.find.user(uid)
......
......@@ -5,6 +5,7 @@ from portal import logger
from portal.rest import rest_helpers
from portal.rest.schemas import SCHEMAS
from portal.rest.custom_resource import CustomResource
from portal.tools.decorators import access_log
workers_namespace = Namespace('workers')
log = logger.get_logger(__name__)
......@@ -21,6 +22,7 @@ class WorkerList(CustomResource):
return SCHEMAS.dump('workers', workers_list)
@jwt_required
@access_log
# @workers_namespace.response(201, 'Created worker', model=worker_schema)
@workers_namespace.response(403, 'Not allowed to add worker')
def post(self):
......@@ -44,6 +46,7 @@ class WorkerResource(CustomResource):
return SCHEMAS.dump('worker', worker)
@jwt_required
@access_log
@workers_namespace.response(204, 'Worker deleted')
@workers_namespace.response(403, 'Not allowed to delete worker')
def delete(self, wid: str):
......@@ -53,13 +56,12 @@ class WorkerResource(CustomResource):
return '', 204
@jwt_required
@access_log
@workers_namespace.response(204, 'Worker updated')
@workers_namespace.response(403, 'Not allowed to update worker')
def put(self, wid: str):
self.permissions().require.sysadmin_or_self(wid)
data = rest_helpers.parse_request_data(action='update', resource='worker', partial=True)
worker = self.find.worker(wid)
self.rest.workers(worker).update(**data)
return '', 204
......@@ -109,13 +109,13 @@ class AuthService:
return False
return True
def find_client(self) -> Client:
def find_client(self, throw=True) -> Client:
identifier = get_jwt_identity()
return self._find_client_helper(identifier)
return self._find_client_helper(identifier, throw=throw)
def _find_client_helper(self, identifier: str) -> Client:
def _find_client_helper(self, identifier: str, throw=True) -> Client:
log.trace(f"[FIND] Finding client using identifier: {identifier}")
client = self._rest_service.find.client(identifier, throws=False)
if not client:
if not client and throw:
raise errors.UnauthorizedError(f"[LOGIN] Unknown client identifier {identifier}.")
return client
......@@ -35,7 +35,7 @@ class CourseService(GeneralService):
course = Course()
self._entity = course
self._set_data(entity=course, **data)
log.debug(f"[CREATE] Course {course.log_name}: {course}")
log.debug(f"[CREATE] Course {course.log_name} by {self.client_name}: {course}")
return course
def copy_course(self, target: Course, config: dict) -> Course:
......@@ -59,7 +59,8 @@ class CourseService(GeneralService):
self._rest_service.projects(project).copy_project(target)
self.write_entity(target)
log.info(f"[IMPORT] From {self.course.log_name} to {target.log_name}.")
log.info(f"[IMPORT] From {self.course.log_name} "
f"to {target.log_name} by {self.client_name}.")
return target
......@@ -73,7 +74,8 @@ class CourseService(GeneralService):
"""
self.course.notes_access_token = token
self.write_entity(self.course)
log.info(f"[UPDATE] Notes access token {self.course.log_name}: {token}")
log.info(f"[UPDATE] Notes access token {self.course.log_name} "
f"by {self.client_name}: {token}")
return self.course
def get_clients_filtered(self, groups: List[str], roles: List[str], client_type=None) -> List[User]:
......
......@@ -34,7 +34,13 @@ class GeneralService:
@property
def client(self):
return self._rest_service.auth.client
return self._rest_service.auth.find_client(throw=False)
@property
def client_name(self):
if self.client:
return self.client.log_name
return 'Unknown client'
@property
def request(self) -> flask.Request:
......@@ -57,7 +63,8 @@ class GeneralService:
return self._entity_klass.query.all()
def delete(self):
log.info(f"[DELETE] {self._entity_klass.__name__} {self._entity.log_name}")
log.info(f"[DELETE] {self._entity_klass.__name__} "
f"{self._entity.log_name} by {self.client_name}")
self.delete_entity(self._entity)
def update(self, **data):
......@@ -69,7 +76,8 @@ class GeneralService:
Returns: Updated instance instance
"""
self._set_data(self._entity, **data)
log.info(f"[UPDATE] {self._entity_klass.__name__} {self._entity.log_name}: {self._entity}.")
log.info(f"[UPDATE] {self._entity_klass.__name__} {self._entity.log_name} "
f"by {self.client_name}: {self._entity}.")
return self._entity
def update_entity(self, entity, data: dict, allowed: list, write: bool = True):
......@@ -97,7 +105,7 @@ class GeneralService:
Args:
entity(db.Model): Database entity instance
"""
log.trace(f"[WRITE] Entity {entity.__class__.__name__}: {entity}")
log.trace(f"[WRITE] Entity {entity.__class__.__name__} by {self.client_name}: {entity}")
self.db_session.add(entity)
self.db_session.commit()
......@@ -107,7 +115,7 @@ class GeneralService:
Args:
entity(db.Model): Database entity instance
"""
log.trace(f"[DELETE] Entity {entity.__class__.__name__}: {entity}")
log.trace(f"[DELETE] Entity {entity.__class__.__name__} by {self.client_name}: {entity}")
self.db_session.delete(entity)
self.db.session.commit()
......@@ -118,7 +126,8 @@ class GeneralService:
Returns: Instance of the same object
"""
log.trace(f"[REFRESH] Entity {entity.__class__.__name__}: {entity}")
log.trace(f"[REFRESH] Entity {entity.__class__.__name__} "
f"by {self.client_name}: {entity}")
self.db.session.refresh(entity)
return entity
......@@ -128,7 +137,8 @@ class GeneralService:
entity: Any db model
Returns: Instance of the same object
"""
log.trace(f"[EXPIRE] Entity {entity.__class__.__name__}: {entity}")
log.trace(f"[EXPIRE] Entity {entity.__class__.__name__}"
f"by {self.client_name}: {entity}")
self.db_session.expire(entity)
return entity
......
......@@ -35,7 +35,8 @@ class GroupService(GeneralService):
Returns: the new group
"""
source = self.group
log.info(f"[COPY] Group: {source.log_name} to course {target.log_name} "
log.info(f"[COPY] Group: {source.log_name} to course {target.log_name}"
f" by {self.client_name} "
f"with config: {with_users}")
new_name = get_new_name(source, target)
new_group = Group(target, codename=new_name)
......@@ -62,7 +63,7 @@ class GroupService(GeneralService):
new_group = Group(course=course)
self._entity = new_group
self._set_data(new_group, **data)
log.info(f"[CREATE] New group {new_group.log_name}: {new_group}")