Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
Michal Štefánik
ARQMath-eval
Commits
15faa1af
Commit
15faa1af
authored
May 07, 2020
by
Vít Novotný
Browse files
Add random system performance to the leaderboards
parent
5d31c92a
Pipeline
#60221
failed with stage
Changes
24
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
README.md
View file @
15faa1af
...
...
@@ -30,7 +30,7 @@ Each task comes with three *subsets*:
#### Using the `train` set to train your supervised system
```
sh
$
pip
install
--force-reinstall
git+https://gitlab.fi.muni.cz/xstefan3/arqmath-eval@0.0.
6
$
pip
install
--force-reinstall
git+https://gitlab.fi.muni.cz/xstefan3/arqmath-eval@0.0.
7
$
python
>>>
from arqmath_eval import get_topics, get_judged_documents, get_ndcg
>>>
...
...
@@ -58,7 +58,7 @@ Here is the documentation of the available evaluation functions:
#### Using the `validation` set to compare various parameters of your system
```
sh
$
pip
install
--force-reinstall
git+https://gitlab.fi.muni.cz/xstefan3/arqmath-eval@0.0.
6
$
pip
install
--force-reinstall
git+https://gitlab.fi.muni.cz/xstefan3/arqmath-eval@0.0.
7
$
python
>>>
from arqmath_eval import get_topics, get_judged_documents
>>>
...
...
ntcir-11-math-2-main/ayetiran/README.md
View file @
15faa1af
...
...
@@ -4,3 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
|
*0.6894*
|
*random*
|
ntcir-11-math-2-main/xluptak4/README.md
View file @
15faa1af
...
...
@@ -4,3 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
|
*0.6894*
|
*random*
|
ntcir-11-math-2-main/xnovot32/README.md
View file @
15faa1af
...
...
@@ -4,3 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
|
*0.6894*
|
*random*
|
ntcir-11-math-2-main/xstefan3/README.md
View file @
15faa1af
...
...
@@ -4,3 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
|
*0.6894*
|
*random*
|
ntcir-12-mathir-arxiv-main/ayetiran/README.md
View file @
15faa1af
...
...
@@ -4,3 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
|
*0.6471*
|
*random*
|
ntcir-12-mathir-arxiv-main/xluptak4/README.md
View file @
15faa1af
...
...
@@ -4,3 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
|
*0.6471*
|
*random*
|
ntcir-12-mathir-arxiv-main/xnovot32/README.md
View file @
15faa1af
...
...
@@ -4,3 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
|
*0.6471*
|
*random*
|
ntcir-12-mathir-arxiv-main/xstefan3/README.md
View file @
15faa1af
...
...
@@ -4,3 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
|
*0.6471*
|
*random*
|
ntcir-12-mathir-math-wiki-formula/ayetiran/README.md
View file @
15faa1af
...
...
@@ -4,3 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
|
*0.7336*
|
*random*
|
ntcir-12-mathir-math-wiki-formula/xluptak4/README.md
View file @
15faa1af
...
...
@@ -4,3 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
|
*0.7336*
|
*random*
|
ntcir-12-mathir-math-wiki-formula/xnovot32/README.md
View file @
15faa1af
...
...
@@ -4,3 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
|
*0.7336*
|
*random*
|
ntcir-12-mathir-math-wiki-formula/xstefan3/README.md
View file @
15faa1af
...
...
@@ -4,3 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
|
*0.7336*
|
*random*
|
scripts/configuration.py
View file @
15faa1af
...
...
@@ -13,7 +13,6 @@ USER_README_HEAD = r'''
This table contains all results for user *%s* in descending order of task
performance. Result names are based on the filenames of the results with
underscores (`_`) replaced with a comma and a space for improved readability.
The random nDCG for this task is %.4f.
| nDCG | Result name |
|------|:------------|
...
...
scripts/evaluate.py
View file @
15faa1af
...
...
@@ -5,6 +5,7 @@ import os.path
import
re
from
pytrec_eval
import
parse_run
from
tqdm
import
tqdm
from
.common
import
get_ndcg
,
get_random_ndcg
from
.configuration
import
TASKS
,
USER_README_HEAD
...
...
@@ -13,18 +14,24 @@ from .configuration import TASKS, USER_README_HEAD
if
__name__
==
'__main__'
:
for
task
in
TASKS
:
random_ndcg
=
get_random_ndcg
(
task
,
'validation'
)
for
user
in
glob
(
os
.
path
.
join
(
task
,
'*'
,
''
)):
users
=
glob
(
os
.
path
.
join
(
task
,
'*'
,
''
))
for
user
in
users
:
user
=
os
.
path
.
normpath
(
user
)
user_name
=
os
.
path
.
basename
(
user
)
user_results
=
[]
for
result
in
glob
(
os
.
path
.
join
(
user
,
'*.tsv'
)):
result_name
=
re
.
sub
(
'_'
,
', '
,
os
.
path
.
basename
(
result
)[:
-
4
])
with
open
(
result
,
'rt'
)
as
f
:
parsed_result
=
parse_run
(
f
)
ndcg
=
get_ndcg
(
parsed_result
,
task
,
'validation'
)
user_results
.
append
((
ndcg
,
result_name
))
with
open
(
os
.
path
.
join
(
user
,
'README.md'
),
'wt'
)
as
f
:
f
.
write
(
USER_README_HEAD
%
(
user_name
,
random_ndcg
))
f
.
write
(
'
\n
'
)
for
ndcg
,
result_name
in
sorted
(
user_results
,
reverse
=
True
):
f
.
write
(
'| %.4f | %s |
\n
'
%
(
ndcg
,
result_name
))
user_results
=
[(
random_ndcg
,
'random'
)]
results
=
glob
(
os
.
path
.
join
(
user
,
'*.tsv'
))
if
results
:
for
result
in
tqdm
(
results
,
desc
=
'Evaluating {} systems'
.
format
(
user
)):
result_name
=
re
.
sub
(
'_'
,
', '
,
os
.
path
.
basename
(
result
)[:
-
4
])
with
open
(
result
,
'rt'
)
as
f
:
parsed_result
=
parse_run
(
f
)
ndcg
=
get_ndcg
(
parsed_result
,
task
,
'validation'
)
user_results
.
append
((
ndcg
,
result_name
))
with
open
(
os
.
path
.
join
(
user
,
'README.md'
),
'wt'
)
as
f
:
f
.
write
(
USER_README_HEAD
%
user_name
)
f
.
write
(
'
\n
'
)
for
ndcg
,
result_name
in
sorted
(
user_results
,
reverse
=
True
):
if
result_name
==
'random'
:
f
.
write
(
'| *%.4f* | *%s* |
\n
'
%
(
ndcg
,
result_name
))
else
:
f
.
write
(
'| %.4f | %s |
\n
'
%
(
ndcg
,
result_name
))
setup.py
View file @
15faa1af
...
...
@@ -5,13 +5,14 @@ from setuptools import setup
setup
(
name
=
'arqmath_eval'
,
version
=
'0.0.
6
'
,
version
=
'0.0.
7
'
,
description
=
'Evaluation of ARQMath systems'
,
packages
=
[
'arqmath_eval'
],
package_dir
=
{
'arqmath_eval'
:
'scripts'
},
install_requires
=
[
'numpy~=1.18.2'
,
'pytrec-eval~=0.4'
,
'tqdm~=4.46.0'
,
],
package_data
=
{
'arqmath_eval'
:
[
...
...
task1-votes/ayetiran/README.md
View file @
15faa1af
...
...
@@ -4,3 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
|
*0.7578*
|
*random*
|
task1-votes/xluptak4/README.md
View file @
15faa1af
...
...
@@ -4,3 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
|
*0.7578*
|
*random*
|
task1-votes/xnovot32/README.md
View file @
15faa1af
...
...
@@ -4,3 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
|
*0.7578*
|
*random*
|
task1-votes/xstefan3/README.md
View file @
15faa1af
...
...
@@ -9,3 +9,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| 0.7602 | sbert, validation, prefix, datav1.0, exid23 |
| 0.7598 | sbert, validation, exid25, no-token-type, datav1.0 |
| 0.7597 | sbert, validation, exid24, token-type, datav1.0 |
|
*0.7578*
|
*random*
|
Prev
1
2
Next
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment