Skip to content
GitLab
Commit 6f9709c3 authored by Barbora Kompišová's avatar Barbora Kompišová Committed by Peter Stanko
Browse files

Secrets refactor

parent 98003104
Pipeline #13042 passed with stage
in 11 minutes and 55 seconds
Hide whitespace changes
Inline Side-by-side
management/data/data_dev.py
View file @ 6f9709c3
......@@ -3,7 +3,7 @@ from flask import Flask
from flask_sqlalchemy import SQLAlchemy
from management.data.shared import DataFactory
from portal.database.models import Course, Review, Submission, SubmissionState
from portal.database.models import Course, Review, Submission, SubmissionState, Secret
from portal.tools import time
......@@ -159,9 +159,12 @@ def init_dev_data(app: Flask, db: SQLAlchemy):
db.session.add_all([review])
# components
factory.create_component(name='executor')
factory.create_component(
name='processing', component_type='processing')
tomorrow = time.current_time() + timedelta(days=1)
executor = factory.create_worker(name='executor', url='some-url')
executor.secrets.append(Secret('executor_secret', 'executor_secret', tomorrow))
processing = factory.create_worker(name='processing', url='some-url')
processing.secrets.append(Secret('processing_secret', 'processing_secret', tomorrow))
db.session.add_all([executor, processing])
# Commit to the DB
db.session.commit()
management/data/data_test.py
View file @ 6f9709c3
......@@ -3,7 +3,7 @@ from flask import Flask
from flask_sqlalchemy import SQLAlchemy
from management.data.shared import DataFactory
from portal.database.models import Review, Submission, SubmissionState
from portal.database.models import Review, Submission, SubmissionState, Secret
from portal.tools import time
......@@ -157,10 +157,13 @@ def init_test_data(app: Flask, db: SQLAlchemy):
db.session.add_all([review])
# components
factory.create_component(name='executor')
factory.create_component(
name='processing', component_type='processing')
# workers
executor = factory.create_worker(name='executor', url="foo/url")
executor.secrets.append(Secret('executor_secret', "executor_secret",
time.current_time() + timedelta(hours=1)))
processing = factory.create_worker(name='processing', url="bar/url")
processing.secrets.append(Secret('processing_secret', "processing_secret",
time.current_time() + timedelta(hours=1)))
db.session.add_all([executor, processing])
# Commit to the DB
db.session.commit()
management/data/shared.py
View file @ 6f9709c3
......@@ -7,7 +7,7 @@ import random
import string
from flask_sqlalchemy import SQLAlchemy
from portal.database.models import Component, Course, Group, Project, ReviewItem, Role, User
from portal.database.models import Worker, Course, Group, Project, ReviewItem, Role, User
log = logging.getLogger(__name__)
......@@ -104,12 +104,8 @@ class DataFactory(object):
log.info(f"[SAMPLE] Create {entity}")
return entity
def create_component(self, name: str, secret: str = None, component_type: str = 'executor',
ip_addr: str = '127.0.0.1', notes: str = None) -> Component:
secret = secret or f"{name}_secret"
notes = notes or f"Component {name} of type {type}"
return self.__create_entity(Component, name=name, secret=secret,
ip_address=ip_addr, notes=notes, component_type=component_type)
def create_worker(self, name: str, url: str) -> Worker:
return self.__create_entity(Worker, name=name, url=url)
def create_course(self, codename, name=None, token=None) -> Course:
name = name or codename
......
migrations/versions/e897fc93ab4e_.py → migrations/versions/38031726c9b7_.py
View file @ 6f9709c3
"""empty message
Revision ID: e897fc93ab4e
Revision ID: 38031726c9b7
Revises:
Create Date: 2018-08-23 17:31:11.993908
Create Date: 2018-09-02 13:40:56.387248
"""
from alembic import op
......@@ -10,7 +10,7 @@ import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'e897fc93ab4e'
revision = '38031726c9b7'
down_revision = None
branch_labels = None
depends_on = None
......@@ -18,17 +18,10 @@ depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('component',
sa.Column('created_at', sa.TIMESTAMP(), nullable=True),
sa.Column('updated_at', sa.TIMESTAMP(), nullable=True),
op.create_table('client',
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('name', sa.String(length=30), nullable=False),
sa.Column('secret', sa.String(length=250), nullable=True),
sa.Column('ip_address', sa.String(length=50), nullable=True),
sa.Column('type', sa.String(length=25), nullable=True),
sa.Column('notes', sa.Text(), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name')
sa.Column('type', sa.Enum('USER', 'WORKER', name='ClientType'), nullable=False),
sa.PrimaryKeyConstraint('id')
)
op.create_table('course',
sa.Column('created_at', sa.TIMESTAMP(), nullable=True),
......@@ -41,20 +34,6 @@ def upgrade():
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('codename', name='course_unique_codename')
)
op.create_table('user',
sa.Column('created_at', sa.TIMESTAMP(), nullable=True),
sa.Column('updated_at', sa.TIMESTAMP(), nullable=True),
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('uco', sa.Integer(), nullable=True),
sa.Column('email', sa.String(length=50), nullable=False),
sa.Column('username', sa.String(length=30), nullable=False),
sa.Column('name', sa.String(length=50), nullable=True),
sa.Column('is_admin', sa.Boolean(), nullable=True),
sa.Column('password_hash', sa.String(length=120), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('email'),
sa.UniqueConstraint('username')
)
op.create_table('group',
sa.Column('created_at', sa.TIMESTAMP(), nullable=True),
sa.Column('updated_at', sa.TIMESTAMP(), nullable=True),
......@@ -92,6 +71,49 @@ def upgrade():
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('course_id', 'codename', name='r_course_unique_name')
)
op.create_table('secret',
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('name', sa.String(length=40), nullable=False),
sa.Column('value', sa.String(length=120), nullable=True),
sa.Column('created_at', sa.TIMESTAMP(), nullable=True),
sa.Column('expires_at', sa.TIMESTAMP(), nullable=False),
sa.Column('client_id', sa.String(length=36), nullable=False),
sa.ForeignKeyConstraint(['client_id'], ['client.id'], ondelete='cascade'),
sa.PrimaryKeyConstraint('id')
)
op.create_table('user',
sa.Column('created_at', sa.TIMESTAMP(), nullable=True),
sa.Column('updated_at', sa.TIMESTAMP(), nullable=True),
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('uco', sa.Integer(), nullable=True),
sa.Column('email', sa.String(length=50), nullable=False),
sa.Column('username', sa.String(length=30), nullable=False),
sa.Column('name', sa.String(length=50), nullable=True),
sa.Column('is_admin', sa.Boolean(), nullable=True),
sa.Column('password_hash', sa.String(length=120), nullable=True),
sa.ForeignKeyConstraint(['id'], ['client.id'], ),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('email'),
sa.UniqueConstraint('username')
)
op.create_table('worker',
sa.Column('created_at', sa.TIMESTAMP(), nullable=True),
sa.Column('updated_at', sa.TIMESTAMP(), nullable=True),
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('name', sa.String(length=30), nullable=False),
sa.Column('url', sa.String(length=250), nullable=False),
sa.Column('tags', sa.Text(), nullable=True),
sa.Column('state', sa.Enum('CREATED', 'READY', 'STOPPED', name='WorkerState'), nullable=False),
sa.ForeignKeyConstraint(['id'], ['client.id'], ),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name')
)
op.create_table('clients_roles',
sa.Column('client_id', sa.String(length=36), nullable=True),
sa.Column('role_id', sa.String(length=36), nullable=True),
sa.ForeignKeyConstraint(['client_id'], ['client.id'], ondelete='cascade'),
sa.ForeignKeyConstraint(['role_id'], ['role.id'], ondelete='cascade')
)
op.create_table('projectConfig',
sa.Column('created_at', sa.TIMESTAMP(), nullable=True),
sa.Column('updated_at', sa.TIMESTAMP(), nullable=True),
......@@ -99,6 +121,7 @@ def upgrade():
sa.Column('project_id', sa.String(length=36), nullable=False),
sa.Column('submissions_cancellation_period', sa.Integer(), nullable=True),
sa.Column('test_files_source', sa.String(length=600), nullable=True),
sa.Column('test_files_commit_hash', sa.String(length=64), nullable=True),
sa.Column('file_whitelist', sa.Text(), nullable=True),
sa.Column('pre_submit_script', sa.Text(), nullable=True),
sa.Column('post_submit_script', sa.Text(), nullable=True),
......@@ -157,6 +180,7 @@ def upgrade():
sa.Column('parameters', sa.Text(), nullable=False),
sa.Column('state', sa.Enum('CREATED', 'READY', 'QUEUED', 'IN_PROGRESS', 'FINISHED', 'CANCELLED', 'ABORTED', 'ARCHIVED', name='SubmissionState'), nullable=False),
sa.Column('note', sa.Text(), nullable=True),
sa.Column('source_hash', sa.String(length=64), nullable=True),
sa.Column('user_id', sa.String(length=36), nullable=False),
sa.Column('project_id', sa.String(length=36), nullable=False),
sa.Column('storage_task_id', sa.String(length=36), nullable=True),
......@@ -171,12 +195,6 @@ def upgrade():
sa.ForeignKeyConstraint(['group_id'], ['group.id'], ondelete='cascade'),
sa.ForeignKeyConstraint(['user_id'], ['user.id'], ondelete='cascade')
)
op.create_table('users_roles',
sa.Column('user_id', sa.String(length=36), nullable=True),
sa.Column('role_id', sa.String(length=36), nullable=True),
sa.ForeignKeyConstraint(['role_id'], ['role.id'], ondelete='cascade'),
sa.ForeignKeyConstraint(['user_id'], ['user.id'], ondelete='cascade')
)
op.create_table('review',
sa.Column('created_at', sa.TIMESTAMP(), nullable=True),
sa.Column('updated_at', sa.TIMESTAMP(), nullable=True),
......@@ -205,16 +223,18 @@ def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('reviewItem')
op.drop_table('review')
op.drop_table('users_roles')
op.drop_table('users_groups')
op.drop_table('submission')
op.drop_table('rolePermissions')
op.drop_table('projects_groups')
op.drop_table('projectConfig')
op.drop_table('clients_roles')
op.drop_table('worker')
op.drop_table('user')
op.drop_table('secret')
op.drop_table('role')
op.drop_table('project')
op.drop_table('group')
op.drop_table('user')
op.drop_table('course')
op.drop_table('component')
op.drop_table('client')
# ### end Alembic commands ###
migrations/versions/9143d16ae835_.py deleted 100644 → 0
View file @ 98003104
"""empty message
Revision ID: 9143d16ae835
Revises: e897fc93ab4e
Create Date: 2018-08-25 13:07:56.545852
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '9143d16ae835'
down_revision = 'e897fc93ab4e'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('projectConfig', sa.Column('test_files_commit_hash', sa.String(length=64), nullable=True))
op.add_column('submission', sa.Column('source_hash', sa.String(length=64), nullable=True))
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('submission', 'source_hash')
op.drop_column('projectConfig', 'test_files_commit_hash')
# ### end Alembic commands ###
portal/database/__init__.py
View file @ 6f9709c3
......@@ -2,7 +2,7 @@
Database layer module
"""
from .models import User, Group, Project, ProjectState, ProjectConfig, Role, Course, Component,\
from .models import User, Group, Project, ProjectState, ProjectConfig, Role, Course, Worker,\
RolePermissions, Submission, SubmissionState, Review, ReviewItem
from .exceptions import PortalDbError
portal/database/models.py
View file @ 6f9709c3
......@@ -9,7 +9,6 @@ from typing import List
from flask_sqlalchemy import BaseQuery
from sqlalchemy import event
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import aliased
from werkzeug.security import check_password_hash, generate_password_hash
from portal import db
......@@ -37,70 +36,76 @@ def _repr(instance) -> str:
return result
class User(db.Model, EntityBase):
"""User entity model
class ClientType(enum.Enum):
"""All known client types
"""
USER = 'user'
WORKER = 'worker'
class Client(db.Model):
"""Client entity model
Attributes:
id: UUID string
uco: University identification number
email: User's email
username: User's username (xlogin for example)
name: User's full name
is_admin: Whether user is administrator
password_hash: Hashed password
submissions: Collection of user's submissions
review_items: Collection of review_items created by user
roles: Collection of all roles the user has
groups: Collection of groups the user belongs to
name: custom name for the secret
type: client type (worker or user)
secrets: a list of secrets for this client
roles: roles associated with this client
owner_id: reference to the enclosing entity of the client
"""
EXCLUDED = ['password_hash', 'submissions', 'review_items']
__tablename__ = 'user'
__tablename__ = 'client'
id = db.Column(db.String(length=36), default=lambda: str(
uuid.uuid4()), primary_key=True)
uco = db.Column(db.Integer)
email = db.Column(db.String(50), unique=True, nullable=False)
username = db.Column(db.String(30), unique=True, nullable=False)
name = db.Column(db.String(50))
is_admin = db.Column(db.Boolean, default=False)
# optional - after password change
password_hash = db.Column(db.String(120), default=None)
type = db.Column(db.Enum(ClientType, name='ClientType'), nullable=False)
secrets = db.relationship('Secret', back_populates='client', uselist=True,
cascade="all, delete-orphan", passive_deletes=True)
submissions = db.relationship("Submission", back_populates="user", cascade="all, delete-orphan",
passive_deletes=True)
review_items = db.relationship("ReviewItem", back_populates="user")
__mapper_args__ = {
'polymorphic_identity': 'client',
'polymorphic_on': type
}
def set_password(self, password: str):
"""Sets password for the user
def __init__(self, client_type: ClientType):
self.type = client_type
def verify_secret(self, secret: str) -> bool:
return any(check_password_hash(item.value, secret) for item in self.secrets)
def query_permissions_for_course(self, course: 'Course'):
"""Returns the query for client's permissions in the course
Args:
password(str): Unhashed password
course(Course): Course instance
Returns: Query for the permissions
"""
self.password_hash = generate_password_hash(password)
permissions = RolePermissions.query.join(RolePermissions.role) \
.filter_by(course=course) \
.join(Role.clients).filter(Client.id == self.id)
return permissions
def verify_password(self, password: str) -> bool:
"""Verifies user's password
def get_permissions_for_course(self, course: 'Course') -> List['RolePermissions']:
"""Gets list of permissions for the course
Args:
password(str): Unhashed password
Returns(bool): True if the password is valid, False otherwise
course(Course): Course instance
Returns(list[Permissions]): List of the Permissions
"""
return check_password_hash(self.password_hash, password)
return self.query_permissions_for_course(course=course).all()
@property
def courses(self) -> list:
"""Gets courses the users is a part of
"""Gets courses the client is a part of
Returns(list): List of courses
"""
return self.query_courses().all()
def query_courses(self) -> BaseQuery:
"""Queries courses the users is signed in
"""Queries courses the client is a part of
Returns(BaseQuery): BaseQuery that can be executed
"""
return Course.query.join(Course.roles).join(
Role.users).filter(User.id == self.id)
Role.clients).filter(Client.id == self.id)
def get_roles_in_course(self, course: 'Course') -> list:
"""Gets user's roles for the course
......@@ -121,7 +126,81 @@ class User(db.Model, EntityBase):
Returns(BaseQuery): BaseQuery that can be executed
"""
return Role.query.filter_by(course=course) \
.join(Role.users).filter(User.id == self.id)
.join(Role.clients).filter(Client.id == self.id)
class Secret(db.Model):
__tablename__ = 'secret'
id = db.Column(db.String(length=36), default=lambda: str(
uuid.uuid4()), primary_key=True)
name = db.Column(db.String(40), nullable=False)
value = db.Column(db.String(120))
created_at = db.Column(db.TIMESTAMP, default=db.func.now())
expires_at = db.Column(db.TIMESTAMP, nullable=False)
client_id = db.Column(db.String(36), db.ForeignKey(
'client.id', ondelete='cascade'), nullable=False)
client = db.relationship(
"Client", back_populates="secrets", uselist=False)
def __init__(self, name: str, value: str, expires_at): # TODO: check date/timestamp
self.name = name
self.value = generate_password_hash(value)
self.expires_at = expires_at
class User(EntityBase, Client):
"""User entity model
Attributes:
id: UUID string
uco: University identification number
email: User's email
username: User's username (xlogin for example)
name: User's full name
is_admin: Whether user is administrator
client: the client part of this user (secrets, roles)
password_hash: Hashed password
submissions: Collection of user's submissions
review_items: Collection of review_items created by user
groups: Collection of groups the user belongs to
"""
EXCLUDED = ['password_hash', 'submissions', 'review_items']
__tablename__ = 'user'
id = db.Column(db.String(length=36), db.ForeignKey('client.id'), default=lambda: str(
uuid.uuid4()), primary_key=True)
uco = db.Column(db.Integer)
email = db.Column(db.String(50), unique=True, nullable=False)
username = db.Column(db.String(30), unique=True, nullable=False)
name = db.Column(db.String(50))
is_admin = db.Column(db.Boolean, default=False)
# optional - after password change
password_hash = db.Column(db.String(120), default=None)
submissions = db.relationship("Submission", back_populates="user", cascade="all, delete-orphan",
passive_deletes=True)
review_items = db.relationship("ReviewItem", back_populates="user")
__mapper_args__ = {
'polymorphic_identity': ClientType.USER,
}
def set_password(self, password: str):
"""Sets password for the user
Args:
password(str): Unhashed password
"""
self.password_hash = generate_password_hash(password)
def verify_password(self, password: str) -> bool:
"""Verifies user's password
Args:
password(str): Unhashed password
Returns(bool): True if the password is valid, False otherwise
"""
return check_password_hash(self.password_hash, password)
def query_groups_in_course(self, course: 'Course',
project: 'Project' = None) -> BaseQuery:
......@@ -152,58 +231,6 @@ class User(db.Model, EntityBase):
return self.query_groups_in_course(
course=course, project=project).all()
def query_users_in_group_based_on_role(self, course: 'Course',
role: 'Role', project: 'Project' = None) -> BaseQuery:
"""Queries users for user's group's based on role
Args:
course(Course): Course instance
project(Project): Project Instance
role(Role): Role instance
Returns(BaseQuery): BaseQuery that can be executed
"""
groups = self.query_groups_in_course(
course=course, project=project).subquery()
grp_alias = aliased(Group)
return User.query.join(grp_alias, User.groups) \
.join(groups).join(User.roles) \
.filter((Role.id == role.id) & (course.id == Role.course_id))
def get_users_in_group_based_on_role(self, course: 'Course', role: 'Role',
project: 'Project' = None) -> list:
"""Gets users for user's group's based on role
Args:
course(Course): Course instance
project(Project): Project Instance
role(Role): Role instance
Returns(BaseQuery): BaseQuery that can be executed
"""
return self.query_users_in_group_based_on_role(course=course, project=project,
role=role).all()
def query_permissions_for_course(self, course: 'Course'):
"""Returns the query for users permissions in the course
Args:
course(Course): Course instance
Returns: Query for the permissions
"""
permissions = RolePermissions.query.join(RolePermissions.role) \
.filter_by(course=course) \
.join(Role.users).filter(User.id == self.id)
return permissions
def get_permissions_for_course(
self, course: 'Course') -> List['RolePermissions']:
"""Gets list of permissions for the course
Args:
course(Course): Course instance
Returns(list[Permissions]): List of the Permissions
"""
return self.query_permissions_for_course(course=course).all()
def query_projects_by_course(self, course: 'Course') -> BaseQuery:
groups = self.get_groups_in_course(course=course)
pids = []
......@@ -240,6 +267,7 @@ class User(db.Model, EntityBase):
self.email = email
self.username = username
self.is_admin = is_admin
super().__init__(ClientType.USER)
def __repr__(self):
return _repr(self)
......@@ -277,7 +305,7 @@ class Course(db.Model, EntityBase, NamedMixin):
__table_args__ = (
db.UniqueConstraint('codename', name='course_unique_codename'),
)
)
def __init__(self, name: str = None, codename: str = None,
description: str = None) -> None:
......@@ -392,7 +420,7 @@ class Project(db.Model, EntityBase, NamedMixin):
__table_args__ = (
db.UniqueConstraint('course_id', 'codename', name='p_course_unique_name'),
)
)
def state(self, timestamp=time.current_time()) -> ProjectState:
"""Gets project state based on the timestamp
......@@ -600,7 +628,7 @@ class Role(db.Model, EntityBase, NamedMixin):
id: UUID of the role
name: Name of the role
description: Description of the role
users: Users associated with the role
clients: Clients associated with the role
course: Course associated with the role
permissions: Permissions associated with the role
"""
......@@ -608,7 +636,7 @@ class Role(db.Model, EntityBase, NamedMixin):
__tablename__ = 'role'
id = db.Column(db.String(length=36), default=lambda: str(
uuid.uuid4()), primary_key=True)
users = db.relationship("User", secondary="users_roles")
clients = db.relationship("Client", secondary="clients_roles")
course_id = db.Column(db.String(36), db.ForeignKey(
'course.id', ondelete='cascade'), nullable=False)
course = db.relationship("Course", back_populates="roles", uselist=False)
......@@ -629,7 +657,7 @@ class Role(db.Model, EntityBase, NamedMixin):
__table_args__ = (
db.UniqueConstraint('course_id', 'codename', name='r_course_unique_name'),
)
)