Commit 3c14793e authored by stefanik12's avatar stefanik12
Browse files

Merge branch 'master' of https://gitlab.fi.muni.cz/xstefan3/arqmath-eval

parents f492afac b844217e

ntcir-11-math-2-main/README.md

0 → 100644
+5 −0
Original line number Original line Diff line number Diff line 
This table contains the best result for every user on the *ntcir-11-math-2-main* task.



| nDCG | Result name | User |

|:-----|:------------|------|

| *0.6894* | *random* | *xrando42* |

ntcir-12-mathir-arxiv-main/README.md

0 → 100644
+5 −0
Original line number Original line Diff line number Diff line 
This table contains the best result for every user on the *ntcir-12-mathir-arxiv-main* task.



| nDCG | Result name | User |

|:-----|:------------|------|

| *0.6471* | *random* | *xrando42* |

ntcir-12-mathir-math-wiki-formula/README.md

0 → 100644
+5 −0
Original line number Original line Diff line number Diff line 
This table contains the best result for every user on the *ntcir-12-mathir-math-wiki-formula* task.



| nDCG | Result name | User |

|:-----|:------------|------|

| *0.7336* | *random* | *xrando42* |

scripts/configuration.py

+8 −4
Original line number Original line Diff line number Diff line
@@ -4,10 +4,10 @@ from pytrec_eval import parse_qrel, RelevanceEvaluator 








TASK_README_HEAD = r'''

TASK_README_HEAD = r'''

This table contains the best result for every user.

This table contains the best result for every user on the *%s* task.





| nDCG | User | Result name |

| nDCG | Result name | User |

|:-----|------|:------------|

|:-----|:------------|------|

'''.strip()

'''.strip()

USER_README_HEAD = r'''

USER_README_HEAD = r'''

This table contains all results for user *%s* in descending order of task

This table contains all results for user *%s* in descending order of task
@@ -42,8 +42,12 @@ RELEVANCE_JUDGEMENTS = { 
        'ntcir-12-mathir-arxiv-main': 'NTCIR12_Math-qrels_agg-test.dat',

        'ntcir-12-mathir-arxiv-main': 'NTCIR12_Math-qrels_agg-test.dat',

        'ntcir-12-mathir-math-wiki-formula': 'NTCIR12_MathWikiFrm-qrels_agg-test.dat',

        'ntcir-12-mathir-math-wiki-formula': 'NTCIR12_MathWikiFrm-qrels_agg-test.dat',

    },

    },

    'all': {

        'task1-votes.V1.2': 'votes-qrels.V1.2.tsv',

        'task2-topics-formula_ids.V.1.1': 'topics-formula_ids-qrels.V1.1.tsv',

    }

    }

TASKS = list(RELEVANCE_JUDGEMENTS['test'].keys())

}

TASKS = list(RELEVANCE_JUDGEMENTS['validation'].keys())

PARSED_RELEVANCE_JUDGEMENTS = {}

PARSED_RELEVANCE_JUDGEMENTS = {}

EVALUATORS = {}

EVALUATORS = {}

for subset, filenames in RELEVANCE_JUDGEMENTS.items():

for subset, filenames in RELEVANCE_JUDGEMENTS.items():

scripts/evaluate.py

+11 −2
Original line number Original line Diff line number Diff line
@@ -9,7 +9,7 @@ from pytrec_eval import parse_run 
from tqdm import tqdm

from tqdm import tqdm





from .common import get_ndcg, get_random_ndcg

from .common import get_ndcg, get_random_ndcg

from .configuration import TASKS, USER_README_HEAD

from .configuration import TASKS, USER_README_HEAD, TASK_README_HEAD









def evaluate_worker(result_filename):

def evaluate_worker(result_filename):
@@ -24,6 +24,7 @@ if __name__ == '__main__': 
    for task in TASKS:

    for task in TASKS:

        random_ndcg = get_random_ndcg(task, 'validation')

        random_ndcg = get_random_ndcg(task, 'validation')

        users = glob(os.path.join(task, '*', ''))

        users = glob(os.path.join(task, '*', ''))

        task_results = [(random_ndcg, 'random', 'xrando42')]

        for user in users:

        for user in users:

            user = os.path.normpath(user)

            user = os.path.normpath(user)

            user_name = os.path.basename(user)

            user_name = os.path.basename(user)
@@ -37,6 +38,7 @@ if __name__ == '__main__': 
                    with open(os.path.join(user, 'README.md'), 'wt') as f_readme:

                    with open(os.path.join(user, 'README.md'), 'wt') as f_readme:

                        f_readme.write(USER_README_HEAD % user_name)

                        f_readme.write(USER_README_HEAD % user_name)

                        f_readme.write('\n')

                        f_readme.write('\n')

                        task_results.append((*max(user_results), user_name))

                        for ndcg, result_name in sorted(user_results, reverse=True):

                        for ndcg, result_name in sorted(user_results, reverse=True):

                            if result_name == 'random':

                            if result_name == 'random':

                                f_readme.write('| *%.4f* | *%s* |\n' % (ndcg, result_name))

                                f_readme.write('| *%.4f* | *%s* |\n' % (ndcg, result_name))
@@ -48,4 +50,11 @@ if __name__ == '__main__': 
                                f_readme.write(f_legend.read())

                                f_readme.write(f_legend.read())

                        except IOError:

                        except IOError:

                            pass

                            pass

        with open(os.path.join(task, 'README.md'), 'wt') as f_readme:

            f_readme.write(TASK_README_HEAD % task)

            f_readme.write('\n')

            for ndcg, result_name, user_name in sorted(task_results, reverse=True):

                if result_name == 'random':

                    f_readme.write('| *%.4f* | *%s* | *%s* |\n' % (ndcg, result_name, user_name))

                else:

                    f_readme.write('|  %.4f  |  %s  |  %s  |\n' % (ndcg, result_name, user_name))