Commit efea5797 authored by Vít Novotný's avatar Vít Novotný
Browse files

Use nDCG' instead of nDCG for evaluation

parent 8add6374
......@@ -2,7 +2,7 @@ This table contains the best result for every user.
| nDCG | User | Result name |
|:-----|------|:------------|
| 0.5600 | xstefan3 | example, key1=value1, key2=value2, etc |
| 0.5600 | xnovot32 | example, key1=value1, key2=value2, etc |
| 0.5600 | xluptak4 | example, key1=value1, key2=value2, etc |
| 0.5600 | ayetiran | example, key1=value1, key2=value2, etc |
| 0.6413 | xstefan3 | example, key1=value1, key2=value2, etc |
| 0.6413 | xnovot32 | example, key1=value1, key2=value2, etc |
| 0.6413 | xluptak4 | example, key1=value1, key2=value2, etc |
| 0.6413 | ayetiran | example, key1=value1, key2=value2, etc |
......@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
| 0.5600 | example, key1=value1, key2=value2, etc |
| 0.6413 | example, key1=value1, key2=value2, etc |
......@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
| 0.5600 | example, key1=value1, key2=value2, etc |
| 0.6413 | example, key1=value1, key2=value2, etc |
......@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
| 0.5600 | example, key1=value1, key2=value2, etc |
| 0.6413 | example, key1=value1, key2=value2, etc |
......@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
| 0.5600 | example, key1=value1, key2=value2, etc |
| 0.6413 | example, key1=value1, key2=value2, etc |
......@@ -2,7 +2,7 @@ This table contains the best result for every user.
| nDCG | User | Result name |
|:-----|------|:------------|
| 0.2915 | xstefan3 | example, key1=value1, key2=value2, etc |
| 0.2915 | xnovot32 | example, key1=value1, key2=value2, etc |
| 0.2915 | xluptak4 | example, key1=value1, key2=value2, etc |
| 0.2915 | ayetiran | example, key1=value1, key2=value2, etc |
| 0.3311 | xstefan3 | example, key1=value1, key2=value2, etc |
| 0.3311 | xnovot32 | example, key1=value1, key2=value2, etc |
| 0.3311 | xluptak4 | example, key1=value1, key2=value2, etc |
| 0.3311 | ayetiran | example, key1=value1, key2=value2, etc |
......@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
| 0.2915 | example, key1=value1, key2=value2, etc |
| 0.3311 | example, key1=value1, key2=value2, etc |
......@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
| 0.2915 | example, key1=value1, key2=value2, etc |
| 0.3311 | example, key1=value1, key2=value2, etc |
......@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
| 0.2915 | example, key1=value1, key2=value2, etc |
| 0.3311 | example, key1=value1, key2=value2, etc |
......@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
| 0.2915 | example, key1=value1, key2=value2, etc |
| 0.3311 | example, key1=value1, key2=value2, etc |
......@@ -2,7 +2,7 @@ This table contains the best result for every user.
| nDCG | User | Result name |
|:-----|------|:------------|
| 0.5970 | xstefan3 | example, key1=value1, key2=value2, etc |
| 0.5970 | xnovot32 | example, key1=value1, key2=value2, etc |
| 0.5970 | xluptak4 | example, key1=value1, key2=value2, etc |
| 0.5970 | ayetiran | example, key1=value1, key2=value2, etc |
| 0.6527 | xstefan3 | example, key1=value1, key2=value2, etc |
| 0.6527 | xnovot32 | example, key1=value1, key2=value2, etc |
| 0.6527 | xluptak4 | example, key1=value1, key2=value2, etc |
| 0.6527 | ayetiran | example, key1=value1, key2=value2, etc |
......@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
| 0.5970 | example, key1=value1, key2=value2, etc |
| 0.6527 | example, key1=value1, key2=value2, etc |
......@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
| 0.5970 | example, key1=value1, key2=value2, etc |
| 0.6527 | example, key1=value1, key2=value2, etc |
......@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
| 0.5970 | example, key1=value1, key2=value2, etc |
| 0.6527 | example, key1=value1, key2=value2, etc |
......@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
| nDCG | Result name |
|------|:------------|
| 0.5970 | example, key1=value1, key2=value2, etc |
| 0.6527 | example, key1=value1, key2=value2, etc |
# -*- coding:utf-8 -*-
from copy import deepcopy
import numpy as np
from .configuration import PARSED_RELEVANCE_JUDGEMENTS
from .configuration import EVALUATORS, PARSED_RELEVANCE_JUDGEMENTS
def ndcg(parsed_run, task='task1', subset='test'):
evaluator = PARSED_RELEVANCE_JUDGEMENTS[subset][task]
evaluation = evaluator.evaluate(parsed_run)
only_judged_parsed_run = deepcopy(parsed_run)
parsed_relevance_judgements = PARSED_RELEVANCE_JUDGEMENTS[subset][task]
for topic_name, results in parsed_run.items():
if topic_name not in parsed_relevance_judgements:
del only_judged_parsed_run[topic_name]
else:
judgements = parsed_relevance_judgements[topic_name]
for document_name in results.keys():
if document_name not in judgements:
del only_judged_parsed_run[topic_name][document_name]
evaluator = EVALUATORS[subset][task]
evaluation = evaluator.evaluate(only_judged_parsed_run)
ndcg = np.mean([
measures['ndcg']
for topic, measures
......
......@@ -33,8 +33,10 @@ RELEVANCE_JUDGEMENTS = {
}
TASKS = list(RELEVANCE_JUDGEMENTS['test'].keys())
PARSED_RELEVANCE_JUDGEMENTS = {}
EVALUATORS = {}
for subset, filenames in RELEVANCE_JUDGEMENTS.items():
PARSED_RELEVANCE_JUDGEMENTS[subset] = {}
EVALUATORS[subset] = {}
for task, filename in filenames.items():
relevance_judgements_filename = os.path.join(
os.path.dirname(__file__),
......@@ -43,4 +45,5 @@ for subset, filenames in RELEVANCE_JUDGEMENTS.items():
with open(relevance_judgements_filename, 'rt') as f:
parsed_relevance_judgements = parse_qrel(f)
evaluator = RelevanceEvaluator(parsed_relevance_judgements, {'ndcg'})
PARSED_RELEVANCE_JUDGEMENTS[subset][task] = evaluator
PARSED_RELEVANCE_JUDGEMENTS[subset][task] = parsed_relevance_judgements
EVALUATORS[subset][task] = evaluator
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment