Access logging and accountability support

portal/__init__.py

@@ -8,7 +8,7 @@ from typing import Union
 from celery import Celery
 from flask import Flask
 from flask_cors import CORS
-from flask_jwt_extended import JWTManager
+from flask_jwt_extended import JWTManager, get_jwt_identity
 from flask_migrate import Migrate
 from flask_oauthlib.client import OAuth
 from flask_sqlalchemy import SQLAlchemy
@@ -127,7 +127,6 @@ def create_app(environment: str = None):
         from flask import request
         body = request.get_data(as_text=True) if request.data else ''
         app.logger.debug(f"[{request.method}] {request.url} - {body}")
-
     return app
 
 

portal/logger.py

@@ -7,6 +7,18 @@ from logging.config import dictConfig
 
 from portal.tools import paths
 
+
+def get_logger_file(name):
+    return {
+        'level': 'DEBUG',
+        'class': 'logging.handlers.RotatingFileHandler',
+        'formatter': 'verbose',
+        'filename': str(paths.LOG_DIR / f'{name}.log'),
+        'maxBytes': 5000000,  # 5MB
+        'backupCount': 5
+    }
+
+
 FORMATTERS = {
     'verbose': {
         'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
@@ -27,24 +39,23 @@ HANDLERS = {
         'class': 'logging.StreamHandler',
         'formatter': 'colored_console'
     },
-    'file': {
-        'level': 'DEBUG',
-        'class': 'logging.handlers.RotatingFileHandler',
-        'formatter': 'verbose',
-        'filename': str(paths.LOG_DIR / 'portal.log'),
-        'maxBytes': 500000,
-        'backupCount': 5
-    }
+    'portal_file': get_logger_file('portal'),
+    'access_file': get_logger_file('access'),
+    'storage_file': get_logger_file('storage'),
+    'flask_file': get_logger_file('flask')
 }
 
 LOGGERS = {
-    'portal': {'handlers': ['console', 'file'], 'level': 'DEBUG', 'propagate': True},
+    'portal': {'handlers': ['console', 'portal_file'], 'level': 'DEBUG', 'propagate': True},
+    'portal.access_log': {
+        'handlers': ['console', 'access_file'], 'level': 'DEBUG', 'propagate': True
+    },
     'tests': {'handlers': ['console'], 'level': 'DEBUG', 'propagate': True},
     'management': {'handlers': ['console'], 'level': 'INFO', 'propagate': True},
     'app': {'handlers': ['console'], 'level': 'DEBUG', 'propagate': True},
-    'flask': {'handlers': ['console'], 'level': 'DEBUG', 'propagate': True},
+    'flask': {'handlers': ['console', 'flask_file'], 'level': 'DEBUG', 'propagate': True},
     'werkzeug': {'handlers': ['console'], 'level': 'DEBUG', 'propagate': True},
-    'storage': {'handlers': ['console'], 'level': 'INFO', 'propagate': True},
+    'storage': {'handlers': ['console', 'storage_file'], 'level': 'INFO', 'propagate': True},
 }
 
 LOGGING_CONF = {
@@ -84,3 +95,10 @@ def load_config(conf_type=None):
 def get_logger(*args, **kwargs):
     logger = logging.getLogger(*args, **kwargs)
     return logger
+
+
+def get_access_logger(*args, **kwargs):
+    return logging.getLogger('portal.access_log', *args, **kwargs)
+
+
+ACCESS = get_access_logger()

portal/rest/client.py

@@ -6,6 +6,7 @@ from portal.database.models import ClientType
 from portal.rest import rest_helpers
 from portal.rest.custom_resource import CustomResource
 from portal.rest.schemas import SCHEMAS
+from portal.tools.decorators import access_log
 
 client_namespace = Namespace('client')
 clients_namespace = Namespace('clients')
@@ -49,6 +50,7 @@ class ClientSecretsController(CustomResource):
         return SCHEMAS.dump('secrets', client.secrets)
 
     @jwt_required
+    @access_log
     # @workers_namespace.response(201, 'Created worker secret', model=secret_schema)
     def post(self, cid: str):
         self.permissions.require.sysadmin_or_self(cid)
@@ -72,6 +74,7 @@ class ClientSecretController(CustomResource):
         return SCHEMAS.dump('secret', secret)
 
     @jwt_required
+    @access_log
     @clients_namespace.response(204, 'Client secret deleted')
     def delete(self, cid: str, sid: str):
         self.rest.permissions.require.sysadmin_or_self(cid)
@@ -81,6 +84,7 @@ class ClientSecretController(CustomResource):
         return '', 204
 
     @jwt_required
+    @access_log
     @clients_namespace.response(204, 'Client secret updated')
     @clients_namespace.response(403, 'Not allowed to update client secret')
     def put(self, cid: str, sid: str):

portal/rest/courses.py

@@ -9,6 +9,7 @@ from portal.rest.custom_resource import CustomResource
 from portal.rest.schemas import SCHEMAS
 from portal.service.errors import ForbiddenError, PortalAPIError
 from portal.service.filters import filter_course_dump
+from portal.tools.decorators import access_log
 
 courses_namespace = Namespace('courses')
 log = logger.get_logger(__name__)
@@ -29,6 +30,7 @@ class CourseList(CustomResource):
     @jwt_required
     # @courses_namespace.response(200, 'Created course', model=course_schema)
     # @courses_namespace.response(403, 'Not allowed to create course', model=course_schema)
+    @access_log
     def post(self):
         self.permissions.require.sysadmin()
 
@@ -62,6 +64,7 @@ class CourseResource(CustomResource):
         raise ForbiddenError(client=client)
 
     @jwt_required
+    @access_log
     @courses_namespace.response(204, 'Course deleted')
     @courses_namespace.response(403, 'Not allowed to delete course')
     def delete(self, cid: str):
@@ -71,6 +74,7 @@ class CourseResource(CustomResource):
         return '', 204
 
     @jwt_required
+    @access_log
     @courses_namespace.response(204, 'Course updated')
     @courses_namespace.response(403, 'Not allowed to update course')
     def put(self, cid: str):
@@ -98,6 +102,7 @@ class CourseNotesToken(CustomResource):
         return course.notes_access_token
 
     @jwt_required
+    @access_log
     @courses_namespace.response(204, 'Course\'s notes access token updated')
     @courses_namespace.response(
         403, 'Not allowed to update course\'s notes access token')
@@ -117,6 +122,7 @@ class CourseNotesToken(CustomResource):
 @courses_namespace.response(404, 'Course not found')
 class CourseImport(CustomResource):
     @jwt_required
+    @access_log
     # @courses_namespace.response(200, 'Course has been imported', model=course_schema)
     @courses_namespace.response(403, 'Not allowed to import course')
     @courses_namespace.response(400, 'Cannot import course to itself.')

portal/rest/groups.py

@@ -6,6 +6,7 @@ from portal import logger
 from portal.rest import rest_helpers
 from portal.rest.custom_resource import CustomResource
 from portal.rest.schemas import SCHEMAS
+from portal.tools.decorators import access_log
 
 groups_namespace = Namespace('')
 
@@ -24,6 +25,7 @@ class GroupsList(CustomResource):
         return SCHEMAS.dump('groups', groups)
 
     @jwt_required
+    @access_log
     # @groups_namespace.response(201, 'Group created', model=group_schema)
     @groups_namespace.response(403, 'Not allowed to create group')
     def post(self, cid: str):
@@ -55,6 +57,7 @@ class GroupResource(CustomResource):
         return SCHEMAS.dump('group', group)
 
     @jwt_required
+    @access_log
     @groups_namespace.response(204, 'Group deleted')
     @groups_namespace.response(403, 'Not allowed to delete group')
     def delete(self, cid: str, gid: str):
@@ -67,6 +70,7 @@ class GroupResource(CustomResource):
         return '', 204
 
     @jwt_required
+    @access_log
     @groups_namespace.response(204, 'Group updated')
     @groups_namespace.response(403, 'Not allowed to update group')
     def put(self, cid: str, gid: str):
@@ -99,6 +103,7 @@ class GroupUsersList(CustomResource):
         return SCHEMAS.dump('users', users)
 
     @jwt_required
+    @access_log
     @groups_namespace.response(204, 'User\'s list updated')
     @groups_namespace.response(403, 'Not allowed to add users to the group')
     def put(self, cid: str, gid: str):
@@ -127,6 +132,7 @@ class GroupUsersList(CustomResource):
 @groups_namespace.response(404, 'User not found')
 class GroupAddOrDeleteSingleUser(CustomResource):
     @jwt_required
+    @access_log
     @groups_namespace.response(204, 'User\'s list updated')
     @groups_namespace.response(403, 'Not allowed to add user to the group')
     def put(self, cid: str, gid: str, uid: str):
@@ -141,6 +147,7 @@ class GroupAddOrDeleteSingleUser(CustomResource):
         return '', 204
 
     @jwt_required
+    @access_log
     @groups_namespace.response(204, 'User\'s list updated')
     @groups_namespace.response(403, 'Not allowed to delete users to the group')
     def delete(self, cid: str, gid: str, uid: str):
@@ -181,6 +188,7 @@ class GroupProjectsList(CustomResource):
 @groups_namespace.response(404, 'Project not found')
 class GroupAddOrDeleteProject(CustomResource):
     @jwt_required
+    @access_log
     @groups_namespace.response(204, 'Projects\'s list updated')
     @groups_namespace.response(403, 'Not allowed to add project to the group')
     def put(self, cid: str, gid: str, pid: str):
@@ -195,6 +203,7 @@ class GroupAddOrDeleteProject(CustomResource):
         return '', 204
 
     @jwt_required
+    @access_log
     @groups_namespace.response(204, 'Projects\'s list updated')
     @groups_namespace.response(
         403, 'Not allowed to delete project from the group')
@@ -215,6 +224,7 @@ class GroupAddOrDeleteProject(CustomResource):
 @groups_namespace.response(404, 'Course not found')
 class GroupImport(CustomResource):
     @jwt_required
+    @access_log
     @groups_namespace.response(201, 'Group import')
     @groups_namespace.response(404, 'Group not found')
     @groups_namespace.response(403, 'Not allowed to import to the group')

portal/rest/projects.py

@@ -8,6 +8,7 @@ from portal.rest import rest_helpers
 from portal.rest.custom_resource import CustomResource
 from portal.rest.schemas import SCHEMAS
 from portal.service import errors
+from portal.tools.decorators import access_log
 
 projects_namespace = Namespace('')  # pylint: disable=invalid-name
 
@@ -27,6 +28,7 @@ class ProjectsList(CustomResource):
         return SCHEMAS.dump('projects', projects)
 
     @jwt_required
+    @access_log
     @projects_namespace.response(404, 'Course not found')
     # @projects_namespace.response(201, 'Project created', model=project_schema)
     def post(self, cid: str):
@@ -58,6 +60,7 @@ class ProjectResource(CustomResource):
         return SCHEMAS.dump('project', project)
 
     @jwt_required
+    @access_log
     @projects_namespace.response(204, 'Project deleted')
     @projects_namespace.response(403, 'Not allowed to delete project')
     def delete(self, cid: str, pid: str):
@@ -70,6 +73,7 @@ class ProjectResource(CustomResource):
         return '', 204
 
     @jwt_required
+    @access_log
     @projects_namespace.response(204, 'Project updated')
     @projects_namespace.response(403, 'Not allowed to update project')
     def put(self, cid: str, pid: str):
@@ -110,6 +114,7 @@ class ProjectConfigResource(CustomResource):
             raise errors.ForbiddenError(perm_service.client)
 
     @jwt_required
+    @access_log
     @projects_namespace.response(204, 'Project config updated')
     def put(self, cid: str, pid: str):
         course = self.find.course(cid)
@@ -129,6 +134,7 @@ class ProjectConfigResource(CustomResource):
 @projects_namespace.response(404, 'Project not found')
 class ProjectTestFilesRefresh(CustomResource):
     @jwt_required
+    @access_log
     @projects_namespace.response(204, 'Project test_files updated')
     def post(self, cid: str, pid: str):
         course = self.find.course(cid)
@@ -160,6 +166,7 @@ class ProjectSubmissions(CustomResource):
         return SCHEMAS.dump('submissions', submissions)
 
     @jwt_required
+    @access_log
     # @projects_namespace.response(201, 'Project submission create', model=submission_schema)
     def post(self, cid: str, pid: str):
         client = self.rest.auth.client

portal/rest/roles.py

@@ -7,6 +7,7 @@ from portal.database.models import ClientType
 from portal.rest import rest_helpers
 from portal.rest.custom_resource import CustomResource
 from portal.rest.schemas import SCHEMAS
+from portal.tools.decorators import access_log
 
 roles_namespace = Namespace('')
 
@@ -24,6 +25,7 @@ class RoleList(CustomResource):
         return SCHEMAS.dump('roles', roles)
 
     @jwt_required
+    @access_log
     # @roles_namespace.response(201, 'Role created', model=role_schema)
     def post(self, cid):
         course = self.find.course(cid)
@@ -52,6 +54,7 @@ class RoleResource(CustomResource):
         return SCHEMAS.dump('role', role)
 
     @jwt_required
+    @access_log
     @roles_namespace.response(204, 'Deleted role')
     def delete(self, cid, rid):
         course = self.find.course(cid)
@@ -63,6 +66,7 @@ class RoleResource(CustomResource):
         return '', 204
 
     @jwt_required
+    @access_log
     @roles_namespace.response(204, 'Role updated')
     def put(self, cid: str, rid: str):
         course = self.find.course(cid)
@@ -94,6 +98,7 @@ class RolePermissions(CustomResource):
         return SCHEMAS.dump('permissions', role.permissions)
 
     @jwt_required
+    @access_log
     # @roles_namespace.response(204, 'Update permissions for course', model=permissions_schema)
     def put(self, cid: str, rid: str):
         course = self.find.course(cid)
@@ -127,6 +132,7 @@ class RoleUsersList(CustomResource):
         return SCHEMAS.dump('users', users)
 
     @jwt_required
+    @access_log
     @roles_namespace.response(204, 'Updates users membership in role')
     def put(self, cid: str, rid: str):
         course = self.find.course(cid)
@@ -154,6 +160,7 @@ class RoleUsersList(CustomResource):
 @roles_namespace.response(404, 'Client not found')
 class RoleClient(CustomResource):
     @jwt_required
+    @access_log
     @roles_namespace.response(204, 'Adds permissions to role')
     def put(self, cid: str, rid: str, clid: str):
         course = self.find.course(cid)
@@ -166,6 +173,7 @@ class RoleClient(CustomResource):
         return '', 204
 
     @jwt_required
+    @access_log
     @roles_namespace.response(204, 'Removes permissions from role')
     def delete(self, cid: str, rid: str, clid: str):
         course = self.find.course(cid)

portal/rest/submissions.py

@@ -5,6 +5,7 @@ from portal import logger, storage
 from portal.rest import rest_helpers
 from portal.rest.custom_resource import CustomResource
 from portal.rest.schemas import SCHEMAS
+from portal.tools.decorators import access_log
 
 submissions_namespace = Namespace('submissions')
 
@@ -33,6 +34,7 @@ class SubmissionResource(CustomResource):
         return SCHEMAS.dump('submission', submission)
 
     @jwt_required
+    @access_log
     @submissions_namespace.response(204, 'Submission deleted')
     def delete(self, sid: str):
         self.permissions.require.sysadmin()
@@ -54,6 +56,7 @@ class SubmissionState(CustomResource):
         return SCHEMAS.dump('submission', submission)
 
     @jwt_required
+    @access_log
     @submissions_namespace.response(204, 'Submission state updated')
     def put(self, sid: str):
         client = self.rest.auth.client
@@ -169,6 +172,7 @@ class SubmissionResultFiles(CustomResource):
         return service.send_file_or_zip(storage_entity)
 
     @jwt_required
+    @access_log
     def post(self, sid: str):
         submission = self.find.submission(sid)
         # authorization
@@ -185,6 +189,7 @@ class SubmissionResultFiles(CustomResource):
 @submissions_namespace.response(404, 'Submissions not found')
 class SubmissionResubmit(CustomResource):
     @jwt_required
+    @access_log
     # @submissions_namespace.response(201, 'New submission', model=submission_schema)