Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
Kontr 2.0
Portal API Backend
Commits
ce1d7678
Verified
Commit
ce1d7678
authored
Apr 28, 2019
by
Peter Stanko
Browse files
Update to move dirs
parent
500de1b3
Pipeline
#31699
passed with stage
in 7 minutes and 20 seconds
Changes
7
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
app.py
View file @
ce1d7678
...
...
@@ -3,6 +3,8 @@ Main application module
- Creates instance of the flask app named app
- registers commands
"""
from
pathlib
import
Path
from
portal.logger
import
Logging
Logging
().
load_config
()
...
...
@@ -54,7 +56,7 @@ def cli_mgmt_arch_submissions():
@
management_cli
.
command
(
'shell'
)
def
cli_mgm
t_arch_submissions
():
def
cli_mgm
g_shell
():
print
(
"[MGMT] Starting interactive shell"
)
with
app
.
app_context
():
import
IPython
...
...
@@ -64,6 +66,20 @@ def cli_mgmt_arch_submissions():
IPython
.
embed
()
@
management_cli
.
command
(
'load-script'
,
help
=
'DANGER ZONE - load script and '
'execute it with app context'
)
@
click
.
argument
(
'file'
)
def
cli_mgmt_load_script
(
file
):
print
(
"[MGMT] loading script"
)
with
app
.
app_context
():
fp
=
Path
(
file
).
absolute
()
if
fp
.
exists
():
content
=
fp
.
read_text
(
'utf-8'
)
exec
(
content
)
else
:
print
(
f
"[ERR] File not exists:
{
fp
}
"
)
@
management_cli
.
command
(
'delete-cancelled'
)
def
cli_mgmt_arch_submissions
():
print
(
"[MGMT] Deleting cancelled submissions - NOW"
)
...
...
hack/updates/transform_storage_dirs.py
View file @
ce1d7678
import
shutil
from
pathlib
import
Path
from
portal
import
storage_wrapper
from
portal.database
import
models
UPDATE_NAME
=
"transform_storage_dirs"
DEPENDS_ON
=
[]
def
change_dirs
(
entity
,
base_path
:
Path
):
old_path
=
base_path
/
entity
.
id
new_path
=
base_path
/
entity
.
storage_dirname
if
old_path
.
exists
():
print
(
f
"[MOVE] FROM
{
old_path
}
to
{
new_path
}
"
)
shutil
.
move
(
old_path
,
new_path
)
def
change_project_dirs
():
print
(
"[UPDATE] Processing projects"
)
tf_path
=
storage_wrapper
.
test_files
.
path
for
project
in
models
.
Project
.
query
.
all
():
change_dirs
(
project
,
tf_path
)
def
change_submission_dirs
():
print
(
"[UPDATE] Processing submissions"
)
source_path
=
storage_wrapper
.
submissions
.
path
results_path
=
storage_wrapper
.
results
.
path
for
submission
in
models
.
Submission
.
query
.
all
():
change_dirs
(
submission
,
source_path
)
change_dirs
(
submission
,
results_path
)
change_project_dirs
()
change_submission_dirs
()
portal/async_celery/submission_processor.py
View file @
ce1d7678
...
...
@@ -75,12 +75,12 @@ class SubmissionProcessor:
log
.
info
(
f
"[ASYNC] Uploading submission:
{
self
.
submission
.
log_name
}
with
{
file_params
}
"
)
updated_entity
:
UploadedEntity
=
self
.
storage
.
\
submissions
.
create
(
dirname
=
self
.
submission
.
id
,
**
file_params
)
submissions
.
create
(
dirname
=
self
.
submission
.
storage_dirname
,
**
file_params
)
self
.
submission_store_ended
(
version
=
updated_entity
.
version
)
def
clone
(
self
,
target
):
log
.
info
(
f
"[ASYNC] Cloning submission:
{
self
.
submission
.
log_name
}
to
{
target
.
log_name
}
"
)
self
.
storage
.
submissions
.
clone
(
self
.
submission
.
id
,
target
.
id
)
self
.
storage
.
submissions
.
clone
(
self
.
submission
.
storage_dirname
,
target
.
id
)
self
.
submission_store_ended
(
version
=
self
.
submission
.
source_hash
)
def
send_to_worker
(
self
):
...
...
@@ -97,7 +97,7 @@ class SubmissionProcessor:
def
upload_result
(
self
,
path
,
file_params
):
log
.
info
(
f
"[ASYNC] Uploading result for the submission "
f
"
{
self
.
submission
.
log_name
}
with
{
file_params
}
"
)
self
.
storage
.
results
.
create
(
dirname
=
self
.
submission
.
id
,
**
file_params
)
self
.
storage
.
results
.
create
(
dirname
=
self
.
submission
.
storage_dirname
,
**
file_params
)
Path
(
path
).
unlink
()
self
.
reset_task_id
(
SubmissionState
.
FINISHED
)
...
...
portal/async_celery/tasks.py
View file @
ce1d7678
...
...
@@ -42,9 +42,7 @@ def archive_submission(submission_id: str):
@
celery_app
.
task
(
name
=
'upload-results-to-storage'
)
def
upload_results_to_storage
(
new_submission_id
:
str
,
path
:
str
):
path
=
str
(
path
)
find_service
=
FindService
()
new_submission
=
find_service
.
submission
(
new_submission_id
)
new_submission
=
FindService
().
submission
(
new_submission_id
)
log
.
info
(
f
"[SUBMIT] Processing results - upload to the storage for "
f
"
{
new_submission
.
log_name
}
:
{
path
}
"
)
SubmissionsService
(
new_submission
).
upload_results_to_storage
(
path
)
...
...
portal/database/models.py
View file @
ce1d7678
...
...
@@ -444,12 +444,13 @@ class Project(db.Model, EntityBase, NamedMixin):
def
state
(
self
)
->
ProjectState
:
return
self
.
get_state_by_timestamp
()
def
get_state_by_timestamp
(
self
,
timestamp
=
time
.
current_time
()
)
->
ProjectState
:
def
get_state_by_timestamp
(
self
,
timestamp
=
None
)
->
ProjectState
:
"""Gets project state based on the timestamp
Args:
timestamp: Time for which the state should be calculated
Returns:
"""
"""
timestamp
=
timestamp
or
time
.
current_time
()
if
not
(
self
.
config
.
submissions_allowed_from
or
self
.
config
.
submissions_allowed_to
or
self
.
config
.
archive_from
):
...
...
@@ -862,7 +863,7 @@ class Submission(db.Model, EntityBase):
return
f
"
{
self
.
project
.
namespace
}
/
{
self
.
user
.
codename
}
/
{
self
.
created_at
}
"
@
hybrid_property
def
storage_dirname
(
self
):
def
storage_dirname
(
self
)
->
str
:
created
=
time
.
simple_fmt
(
self
.
created_at
)
return
f
"
{
self
.
user
.
username
}
_
{
created
}
_
{
self
.
course
.
codename
}
_
{
self
.
project
.
codename
}
"
...
...
portal/facade/submissions_facade.py
View file @
ce1d7678
import
logging
from
werkzeug.datastructures
import
FileStorage
from
portal.async_celery
import
tasks
from
portal.database
import
Project
,
Submission
,
SubmissionState
,
User
from
portal.facade.general_facade
import
GeneralCRUDFacade
...
...
@@ -81,10 +83,10 @@ class SubmissionsFacade(GeneralCRUDFacade):
def
upload_results_to_storage
(
self
,
submission
:
Submission
):
log
.
info
(
f
"[UPLOAD] Uploading results to storage for "
f
"
{
submission
.
log_name
}
by
{
self
.
client_name
}
"
)
file
=
self
.
_request
.
files
[
'file'
]
file
:
FileStorage
=
self
.
_request
.
files
[
'file'
]
submission_service
:
SubmissionsService
=
self
.
_service
(
submission
)
return
tasks
.
upload_results_to_storage
.
delay
(
submission
.
id
,
submission_service
.
get_upload_file_path
(
file
)
)
path
=
submission_service
.
get_upload_file_path
(
file
)
return
tasks
.
upload_results_to_storage
.
delay
(
submission
.
id
,
path
)
def
copy_submission
(
self
,
source_submission
:
Submission
,
**
params
):
"""Copies a submission. Used at resubmitting
...
...
portal/service/submissions.py
View file @
ce1d7678
...
...
@@ -6,6 +6,7 @@ import json
import
logging
from
pathlib
import
Path
from
werkzeug.datastructures
import
FileStorage
from
werkzeug.utils
import
secure_filename
from
portal
import
storage_wrapper
...
...
@@ -39,10 +40,11 @@ class SubmissionsService(GeneralService):
SubmissionsService
(
submission
).
archive
()
@
classmethod
def
get_upload_file_path
(
cls
,
file
)
->
str
:
upload_file_is_allowed
(
file
)
path
=
upload_files_to_storage
(
file
)
return
str
(
path
)
def
get_upload_file_path
(
cls
,
file
:
FileStorage
)
->
str
:
if
upload_file_is_allowed
(
file
):
path
=
upload_files_to_storage
(
file
)
return
str
(
path
)
return
None
def
processor
(
self
)
->
SubmissionProcessor
:
return
SubmissionProcessor
(
self
.
submission
)
...
...
@@ -152,8 +154,7 @@ class SubmissionsService(GeneralService):
"""
self
.
processor
().
revoke_task
()
def
upload_results_to_storage
(
self
,
file
):
path
=
self
.
get_upload_file_path
(
file
)
def
upload_results_to_storage
(
self
,
path
):
processor
=
self
.
processor
()
file_params
=
dict
(
source
=
dict
(
url
=
path
,
type
=
'zip'
))
processor
.
upload_result
(
path
=
path
,
file_params
=
file_params
)
...
...
@@ -224,7 +225,7 @@ def nonempty_intersection(provided: list, required: list):
return
list
(
set
(
provided
)
&
set
(
required
))
def
upload_files_to_storage
(
file
):
def
upload_files_to_storage
(
file
:
FileStorage
):
filename
=
secure_filename
(
file
.
filename
)
uploads
=
storage_wrapper
.
results
.
workspace_path
/
'uploads'
if
not
uploads
.
exists
():
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment