Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
Kontr 2.0
Portal API Backend
Commits
0f276bfb
Unverified
Commit
0f276bfb
authored
Sep 23, 2018
by
Peter Stanko
Browse files
Tests for submission files - WIP
parent
da8f1fc4
Pipeline
#13447
passed with stage
in 25 minutes and 12 seconds
Changes
10
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
management/data/__init__.py
View file @
0f276bfb
...
...
@@ -102,6 +102,15 @@ class DataManagement(object):
log
.
debug
(
f
'[DATA] Created course:
{
course
.
log_name
}
'
)
return
course
def
create_project
(
self
,
course
:
str
,
name
):
with
self
.
app
.
app_context
():
course
=
self
.
rest
.
find
.
course
(
course
)
role
=
self
.
creator
.
scaffold_project
(
course
,
name
)
self
.
db
.
session
.
commit
()
log
.
debug
(
f
'[DATA] Created role:
{
role
.
log_name
}
'
)
return
role
def
create_role
(
self
,
course_name
:
str
,
role_type
:
str
,
name
:
str
)
->
Role
:
"""Creates role in the course based on type
Args:
...
...
@@ -175,4 +184,4 @@ class DataManagement(object):
user
=
self
.
rest
.
find
.
user
(
name
)
user
.
secrets
.
append
(
Secret
(
name
=
'user-cli-secret'
,
value
=
secret
))
log
.
debug
(
f
'[DATA] Created secret for user:
{
user
.
log_name
}
'
)
self
.
db
.
session
.
commit
()
self
.
db
.
session
.
commit
()
\ No newline at end of file
management/data/shared.py
View file @
0f276bfb
...
...
@@ -4,10 +4,12 @@ Factory to create sample data
import
random
import
string
from
flask_sqlalchemy
import
SQLAlchemy
from
portal
import
logger
from
portal.database.models
import
Worker
,
Course
,
Group
,
Project
,
ReviewItem
,
Role
,
User
from
portal.database.models
import
Course
,
Group
,
Project
,
ReviewItem
,
Role
,
Submission
,
User
,
\
Worker
log
=
logger
.
get_logger
(
__name__
)
...
...
@@ -152,6 +154,9 @@ class DataFactory(object):
project
.
set_config
(
**
config
)
return
project
def
create_submission
(
self
,
project
,
user
,
**
kwargs
):
return
self
.
__create_entity
(
Submission
,
project
=
project
,
user
=
user
,
**
kwargs
)
def
create_review_item
(
self
,
review
,
author
,
file
=
"main.c"
,
line
=
1
,
content
=
"problem here"
):
return
self
.
__create_entity
(
ReviewItem
,
review
=
review
,
user
=
author
,
file
=
file
,
...
...
@@ -170,3 +175,6 @@ class DataFactory(object):
role
=
self
.
create_role
(
course
=
course
,
name
=
name
,
permissions
=
permissions
)
return
role
def
scaffold_project
(
self
,
course
,
name
):
return
self
.
create_project
(
course
=
course
,
name
=
name
,
codename
=
name
)
portal/async_celery/submission_processor.py
View file @
0f276bfb
import
random
from
pathlib
import
Path
from
typing
import
Optional
from
storage
import
UploadedEntity
...
...
@@ -7,6 +8,7 @@ import portal.tools.worker_client
from
portal
import
logger
,
tools
from
portal.database
import
Project
,
Submission
,
SubmissionState
,
Worker
from
portal.database.models
import
WorkerState
from
portal.service
import
errors
log
=
logger
.
get_logger
(
__name__
)
...
...
@@ -84,7 +86,8 @@ class SubmissionProcessor:
# TODO: implement processing
log
.
info
(
f
"[ASYNC] Sending submission to worker:
{
self
.
submission
.
log_name
}
"
)
worker
=
self
.
schedule_submission_to_worker
()
self
.
execute_submission
(
worker
)
if
worker
:
self
.
execute_submission
(
worker
)
def
upload_result
(
self
,
path
,
file_params
):
log
.
info
(
f
"[ASYNC] Uploading result for the submission "
...
...
@@ -114,7 +117,7 @@ class SubmissionProcessor:
# TODO implement - @mdujava
# STUB: Select initialized worker
def
schedule_submission_to_worker
(
self
)
->
Worker
:
def
schedule_submission_to_worker
(
self
)
->
Optional
[
Worker
]
:
"""Based on the features (worker tags) and preferences in project config
schedule submission for the execution on initialized worker
...
...
@@ -123,6 +126,7 @@ class SubmissionProcessor:
workers
=
self
.
_get_avail_workers
()
if
not
workers
:
self
.
_worker_not_available
()
return
None
worker
=
random
.
choice
(
workers
)
# randomly select a worker
return
worker
...
...
@@ -132,4 +136,3 @@ class SubmissionProcessor:
def
_worker_not_available
(
self
):
log
.
warning
(
f
"[PROC] Worker is no available for submission:
{
self
.
submission
.
log_name
}
"
)
pass
portal/config.py
View file @
0f276bfb
...
...
@@ -80,6 +80,7 @@ class TestConfig(Config):
PORTAL_STORAGE_TEST_FILES_DIR
=
f
"
{
PORTAL_STORAGE_BASE_DIR
}
/test-files"
PORTAL_STORAGE_WORKSPACE_DIR
=
f
"
{
PORTAL_STORAGE_BASE_DIR
}
/workspace"
PORTAL_STORAGE_SUBMISSIONS_DIR
=
f
"
{
PORTAL_STORAGE_BASE_DIR
}
/submissions"
PORTAL_STORAGE_RESULTS_DIR
=
f
"
{
PORTAL_STORAGE_BASE_DIR
}
/results"
PORTAL_ADMIN_USER_USERNAME
=
'admin'
PORTAL_ADMIN_USER_PASSWORD
=
'789789'
EMAIL_BACKEND
=
'tests.utils.email_backend.EmailBackend'
...
...
portal/service/submissions.py
View file @
0f276bfb
...
...
@@ -164,16 +164,18 @@ class SubmissionsService(GeneralService):
def
send_zip
(
self
,
storage_submission
:
entities
.
Submission
):
storage_submission
=
storage_submission
or
self
.
storage_submission
path
=
storage_submission
.
zip_path
klass
=
self
.
storage_submission
.
__class__
.
__name__
if
not
path
.
exists
():
raise
errors
.
PortalAPIError
(
400
,
f
"Requested path does not exist:
{
path
}
"
)
log
.
debug
(
f
"[SEND] Sending zip file
for submission
"
f
"(
{
s
torage_
submission
.
entity_
id
}
):
{
path
}
"
)
log
.
debug
(
f
"[SEND] Sending zip file
{
klass
}
"
f
"(
{
s
elf
.
submission
.
id
}
):
{
path
}
"
)
return
flask
.
send_file
(
str
(
path
),
attachment_filename
=
path
.
name
)
def
send_selected_file
(
self
,
storage_entity
:
entities
.
Submission
,
path_query
:
str
):
storage_entity
=
storage_entity
or
self
.
storage_submission
klass
=
self
.
storage_submission
.
__class__
.
__name__
log
.
debug
(
f
"[SEND] Sending file
{
klass
}
{
self
.
submission
.
log_name
}
:
{
path_query
}
"
)
path
=
storage_entity
.
get
(
path_query
)
log
.
debug
(
f
"[SEND] Sending file for submission (
{
storage_entity
.
entity_id
}
):
{
path
}
"
)
return
flask
.
send_file
(
str
(
path
),
attachment_filename
=
path
.
name
)
def
send_file_or_zip
(
self
,
storage_entity
=
None
):
...
...
tests/conftest.py
View file @
0f276bfb
...
...
@@ -33,3 +33,9 @@ def client(app):
def
rest_service
():
from
portal.service.rest
import
RestService
return
RestService
()
@
pytest
.
fixture
()
def
ent_mocker
(
app
):
from
tests.utils.ent_mocker
import
EntitiesMocker
return
EntitiesMocker
(
app
,
db
=
db
)
tests/rest/test_submission.py
View file @
0f276bfb
...
...
@@ -2,7 +2,7 @@ import json
import
pytest
from
portal.database.models
import
Course
,
Project
,
Review
,
ReviewItem
,
Submission
,
SubmissionState
,
User
,
Group
from
portal.database.models
import
Review
,
ReviewItem
,
Submission
,
SubmissionState
from
.
import
utils
...
...
tests/rest/test_submission_files.py
0 → 100644
View file @
0f276bfb
import
pytest
from
tests.utils.ent_mocker
import
EntitiesMocker
from
.
import
utils
@
pytest
.
fixture
()
def
mocked_submission
(
ent_mocker
:
EntitiesMocker
,
rest_service
):
submission
=
ent_mocker
.
create_submission
()
rest_service
.
submissions
.
write_entity
(
submission
)
ent_mocker
.
create_submission_storage
(
submission
=
submission
)
return
submission
def
expected
(
fpath
):
return
f
'This is a test file:
{
fpath
}
'
def
test_submission_sources_are_available
(
client
,
mocked_submission
):
s
=
mocked_submission
response
=
utils
.
make_request
(
client
,
f
'/submissions/
{
s
.
id
}
/files/sources?path=src/main.c'
)
assert
response
.
status_code
==
200
assert
response
.
data
assert
response
.
data
.
decode
(
'utf-8'
)
==
expected
(
'src/main.c'
)
tests/rest/utils.py
View file @
0f276bfb
import
json
from
datetime
import
datetime
from
flask
import
Response
from
flask.testing
import
FlaskClient
from
portal
import
logger
from
portal.database.models
import
Worker
,
Course
,
Group
,
Project
,
ProjectConfig
,
Review
,
\
ReviewItem
,
Role
,
Submission
,
User
from
portal.database.models
import
Course
,
Group
,
Project
,
ProjectConfig
,
Review
,
ReviewItem
,
Role
,
\
Submission
,
User
,
Worker
DEFAULT_USER_CREDENTIALS
=
json
.
dumps
({
"type"
:
"username_password"
,
...
...
@@ -24,7 +25,7 @@ def extract_data(response: Response) -> dict:
return
data
def
make_request
(
client
:
FlaskClient
,
url
:
str
,
method
:
str
=
'get'
,
def
make_request
(
client
:
FlaskClient
,
url
:
str
,
method
:
str
=
'get'
,
credentials
:
str
=
None
,
headers
:
dict
=
None
,
data
:
str
=
None
)
->
Response
:
""" Creates an authenticated request to an endpoint.
...
...
@@ -69,8 +70,8 @@ def assert_user(expected: User, actual: dict):
def
compare_user
(
expected
:
User
,
actual
:
dict
)
->
bool
:
return
expected
.
id
==
actual
[
'id'
]
and
\
expected
.
uco
==
actual
[
'uco'
]
and
\
expected
.
email
==
actual
[
'email'
]
expected
.
uco
==
actual
[
'uco'
]
and
\
expected
.
email
==
actual
[
'email'
]
def
assert_user_in
(
user_list
:
list
,
user
:
dict
)
->
bool
:
...
...
@@ -87,9 +88,9 @@ def assert_submission(expected: Submission, actual: dict):
def
compare_submission
(
expected
:
Submission
,
actual
:
dict
)
->
bool
:
return
expected
.
id
==
actual
[
'id'
]
\
and
expected
.
state
==
actual
[
'state'
]
\
and
expected
.
project
.
id
==
actual
[
'project'
][
'id'
]
\
and
expected
.
scheduled_for
==
actual
[
'scheduled_for'
]
and
expected
.
state
==
actual
[
'state'
]
\
and
expected
.
project
.
id
==
actual
[
'project'
][
'id'
]
\
and
expected
.
scheduled_for
==
actual
[
'scheduled_for'
]
def
assert_submission_in
(
submission_list
:
list
,
submission
:
dict
):
...
...
@@ -98,8 +99,8 @@ def assert_submission_in(submission_list: list, submission: dict):
def
compare_worker
(
expected
:
Worker
,
actual
:
dict
)
->
bool
:
return
expected
.
id
==
actual
[
'id'
]
and
\
expected
.
url
==
actual
[
'url'
]
and
\
expected
.
name
==
actual
[
'name'
]
expected
.
url
==
actual
[
'url'
]
and
\
expected
.
name
==
actual
[
'name'
]
def
assert_worker_in
(
worker_list
:
list
,
worker
:
dict
):
...
...
@@ -127,7 +128,7 @@ def assert_course(expected: Course, actual: dict):
def
compare_course
(
expected
:
Course
,
actual
:
dict
)
->
bool
:
return
expected
.
id
==
actual
[
'id'
]
and
\
expected
.
codename
==
actual
[
'codename'
]
expected
.
codename
==
actual
[
'codename'
]
def
assert_course_in
(
course_list
:
list
,
course
:
dict
)
->
bool
:
...
...
@@ -142,7 +143,7 @@ def assert_group(expected: Group, actual: dict):
def
compare_group
(
expected
:
Group
,
actual
:
dict
)
->
bool
:
return
expected
.
id
==
actual
[
'id'
]
and
expected
.
name
==
actual
[
'name'
]
\
and
expected
.
course_id
==
actual
[
'course'
][
'id'
]
and
expected
.
course_id
==
actual
[
'course'
][
'id'
]
def
assert_group_in
(
group_list
:
list
,
group
:
dict
)
->
bool
:
...
...
@@ -151,8 +152,8 @@ def assert_group_in(group_list: list, group: dict) -> bool:
def
compare_project
(
expected
:
Project
,
actual
:
dict
)
->
bool
:
return
expected
.
id
==
actual
[
'id'
]
and
\
expected
.
name
==
actual
[
'name'
]
and
\
expected
.
course_id
==
actual
[
'course'
][
'id'
]
expected
.
name
==
actual
[
'name'
]
and
\
expected
.
course_id
==
actual
[
'course'
][
'id'
]
def
assert_project
(
expected
:
Project
,
actual
:
dict
):
...
...
@@ -216,9 +217,9 @@ def assert_role_permissions(expected: Role, actual: dict):
def
compare_role
(
expected
:
Role
,
actual
:
dict
)
->
bool
:
return
expected
.
id
==
actual
[
'id'
]
\
and
expected
.
name
==
actual
[
'name'
]
\
and
expected
.
description
==
actual
[
'description'
]
\
and
expected
.
course_id
==
actual
[
'course'
][
'id'
]
and
expected
.
name
==
actual
[
'name'
]
\
and
expected
.
description
==
actual
[
'description'
]
\
and
expected
.
course_id
==
actual
[
'course'
][
'id'
]
def
assert_role_in
(
role_list
:
list
,
role
:
dict
)
->
bool
:
...
...
@@ -227,11 +228,11 @@ def assert_role_in(role_list: list, role: dict) -> bool:
def
compare_review_items
(
expected
:
ReviewItem
,
actual
:
dict
)
->
bool
:
return
expected
.
id
==
actual
[
'id'
]
\
and
expected
.
review_id
==
actual
[
'review'
][
'id'
]
\
and
expected
.
user_id
==
actual
[
'user'
][
'id'
]
\
and
expected
.
content
==
actual
[
'content'
]
\
and
expected
.
file
==
actual
[
'file'
]
\
and
expected
.
line
==
actual
[
'line'
]
and
expected
.
review_id
==
actual
[
'review'
][
'id'
]
\
and
expected
.
user_id
==
actual
[
'user'
][
'id'
]
\
and
expected
.
content
==
actual
[
'content'
]
\
and
expected
.
file
==
actual
[
'file'
]
\
and
expected
.
line
==
actual
[
'line'
]
def
assert_review_item_in
(
item_list
:
list
,
item
:
dict
)
->
bool
:
...
...
tests/utils/ent_mocker.py
0 → 100644
View file @
0f276bfb
import
secrets
from
pathlib
import
Path
from
storage
import
Entity
from
management.data
import
DataManagement
from
portal.service.rest
import
RestService
class
EntitiesMocker
:
def
__init__
(
self
,
app
,
db
):
self
.
mgmt
=
DataManagement
(
app
=
app
,
db
=
db
)
@
property
def
data
(
self
):
return
self
.
mgmt
.
creator
@
property
def
rest
(
self
)
->
RestService
:
return
self
.
mgmt
.
rest
@
property
def
random_suffix
(
self
):
return
secrets
.
token_urlsafe
(
5
)
def
rand_name
(
self
,
what
):
return
f
'test_
{
what
}
_
{
self
.
random_suffix
}
'
def
create_course
(
self
):
return
self
.
data
.
create_course
(
self
.
rand_name
(
'course'
))
def
create_project
(
self
,
course
=
None
):
course
=
course
or
self
.
create_course
()
return
self
.
data
.
create_project
(
course
,
self
.
rand_name
(
'project'
))
def
create_role
(
self
,
course
=
None
):
course
=
course
or
self
.
create_course
()
return
self
.
data
.
create_role
(
course
,
self
.
rand_name
(
'role'
))
def
create_group
(
self
,
course
=
None
):
course
=
course
or
self
.
create_course
()
return
self
.
data
.
create_role
(
course
,
self
.
rand_name
(
'group'
))
def
create_submission
(
self
,
project
=
None
,
course
=
None
,
user
=
None
):
project
=
project
or
self
.
create_project
(
course
=
course
)
user
=
user
or
self
.
rest
.
find
.
user
(
'admin'
)
return
self
.
data
.
create_submission
(
project
=
project
,
user
=
user
)
def
create_submission_storage
(
self
,
submission
):
storage
=
self
.
rest
.
submissions
.
storage
def
__create_content
(
location
:
Entity
,
*
files
):
path
=
location
.
path
path
.
mkdir
(
parents
=
True
)
self
.
_create_files
(
path
,
*
files
)
location
.
compress
()
subm
=
storage
.
submissions
.
get
(
submission
.
id
)
results
=
storage
.
results
.
get
(
submission
.
id
)
test_files
=
storage
.
test_files
.
get
(
submission
.
project
.
id
)
__create_content
(
subm
,
'src/main.c'
,
'src/foo.c'
)
__create_content
(
results
,
'results.json'
,
'student.json'
)
__create_content
(
test_files
,
'test_main.c'
)
def
_create_files
(
self
,
path
:
Path
,
*
files
):
for
fpath
in
files
:
full_path
:
Path
=
path
/
fpath
if
not
full_path
.
parent
.
exists
():
full_path
.
parent
.
mkdir
(
parents
=
True
)
full_path
.
write_text
(
f
'This is a test file:
{
fpath
}
'
)
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment