Use nDCG' instead of nDCG for evaluation

ntcir-11-math-2-main/README.md

@@ -2,7 +2,7 @@ This table contains the best result for every user.
 
 | nDCG | User | Result name |
 |:-----|------|:------------|
-| 0.5600 | xstefan3 | example, key1=value1, key2=value2, etc |
-| 0.5600 | xnovot32 | example, key1=value1, key2=value2, etc |
-| 0.5600 | xluptak4 | example, key1=value1, key2=value2, etc |
-| 0.5600 | ayetiran | example, key1=value1, key2=value2, etc |
+| 0.6413 | xstefan3 | example, key1=value1, key2=value2, etc |
+| 0.6413 | xnovot32 | example, key1=value1, key2=value2, etc |
+| 0.6413 | xluptak4 | example, key1=value1, key2=value2, etc |
+| 0.6413 | ayetiran | example, key1=value1, key2=value2, etc |

ntcir-11-math-2-main/ayetiran/README.md

@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
 
 | nDCG | Result name |
 |------|:------------|
-| 0.5600 | example, key1=value1, key2=value2, etc |
+| 0.6413 | example, key1=value1, key2=value2, etc |

ntcir-11-math-2-main/xluptak4/README.md

@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
 
 | nDCG | Result name |
 |------|:------------|
-| 0.5600 | example, key1=value1, key2=value2, etc |
+| 0.6413 | example, key1=value1, key2=value2, etc |

ntcir-11-math-2-main/xnovot32/README.md

@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
 
 | nDCG | Result name |
 |------|:------------|
-| 0.5600 | example, key1=value1, key2=value2, etc |
+| 0.6413 | example, key1=value1, key2=value2, etc |

ntcir-11-math-2-main/xstefan3/README.md

@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
 
 | nDCG | Result name |
 |------|:------------|
-| 0.5600 | example, key1=value1, key2=value2, etc |
+| 0.6413 | example, key1=value1, key2=value2, etc |

ntcir-12-mathir-arxiv-main/README.md

@@ -2,7 +2,7 @@ This table contains the best result for every user.
 
 | nDCG | User | Result name |
 |:-----|------|:------------|
-| 0.2915 | xstefan3 | example, key1=value1, key2=value2, etc |
-| 0.2915 | xnovot32 | example, key1=value1, key2=value2, etc |
-| 0.2915 | xluptak4 | example, key1=value1, key2=value2, etc |
-| 0.2915 | ayetiran | example, key1=value1, key2=value2, etc |
+| 0.3311 | xstefan3 | example, key1=value1, key2=value2, etc |
+| 0.3311 | xnovot32 | example, key1=value1, key2=value2, etc |
+| 0.3311 | xluptak4 | example, key1=value1, key2=value2, etc |
+| 0.3311 | ayetiran | example, key1=value1, key2=value2, etc |

ntcir-12-mathir-arxiv-main/ayetiran/README.md

@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
 
 | nDCG | Result name |
 |------|:------------|
-| 0.2915 | example, key1=value1, key2=value2, etc |
+| 0.3311 | example, key1=value1, key2=value2, etc |

ntcir-12-mathir-arxiv-main/xluptak4/README.md

@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
 
 | nDCG | Result name |
 |------|:------------|
-| 0.2915 | example, key1=value1, key2=value2, etc |
+| 0.3311 | example, key1=value1, key2=value2, etc |

ntcir-12-mathir-arxiv-main/xnovot32/README.md

@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
 
 | nDCG | Result name |
 |------|:------------|
-| 0.2915 | example, key1=value1, key2=value2, etc |
+| 0.3311 | example, key1=value1, key2=value2, etc |

ntcir-12-mathir-arxiv-main/xstefan3/README.md

@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
 
 | nDCG | Result name |
 |------|:------------|
-| 0.2915 | example, key1=value1, key2=value2, etc |
+| 0.3311 | example, key1=value1, key2=value2, etc |

ntcir-12-mathir-math-wiki-formula/README.md

@@ -2,7 +2,7 @@ This table contains the best result for every user.
 
 | nDCG | User | Result name |
 |:-----|------|:------------|
-| 0.5970 | xstefan3 | example, key1=value1, key2=value2, etc |
-| 0.5970 | xnovot32 | example, key1=value1, key2=value2, etc |
-| 0.5970 | xluptak4 | example, key1=value1, key2=value2, etc |
-| 0.5970 | ayetiran | example, key1=value1, key2=value2, etc |
+| 0.6527 | xstefan3 | example, key1=value1, key2=value2, etc |
+| 0.6527 | xnovot32 | example, key1=value1, key2=value2, etc |
+| 0.6527 | xluptak4 | example, key1=value1, key2=value2, etc |
+| 0.6527 | ayetiran | example, key1=value1, key2=value2, etc |

ntcir-12-mathir-math-wiki-formula/ayetiran/README.md

@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
 
 | nDCG | Result name |
 |------|:------------|
-| 0.5970 | example, key1=value1, key2=value2, etc |
+| 0.6527 | example, key1=value1, key2=value2, etc |

ntcir-12-mathir-math-wiki-formula/xluptak4/README.md

@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
 
 | nDCG | Result name |
 |------|:------------|
-| 0.5970 | example, key1=value1, key2=value2, etc |
+| 0.6527 | example, key1=value1, key2=value2, etc |

ntcir-12-mathir-math-wiki-formula/xnovot32/README.md

@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
 
 | nDCG | Result name |
 |------|:------------|
-| 0.5970 | example, key1=value1, key2=value2, etc |
+| 0.6527 | example, key1=value1, key2=value2, etc |

ntcir-12-mathir-math-wiki-formula/xstefan3/README.md

@@ -4,4 +4,4 @@ underscores (`_`) replaced with a comma and a space for improved readability.
 
 | nDCG | Result name |
 |------|:------------|
-| 0.5970 | example, key1=value1, key2=value2, etc |
+| 0.6527 | example, key1=value1, key2=value2, etc |

scripts/common.py

 # -*- coding:utf-8 -*-
 
+from copy import deepcopy
+
 import numpy as np
 
-from .configuration import PARSED_RELEVANCE_JUDGEMENTS
+from .configuration import EVALUATORS, PARSED_RELEVANCE_JUDGEMENTS
 
 
 def ndcg(parsed_run, task='task1', subset='test'):
-    evaluator = PARSED_RELEVANCE_JUDGEMENTS[subset][task]
-    evaluation = evaluator.evaluate(parsed_run)
+    only_judged_parsed_run = deepcopy(parsed_run)
+    parsed_relevance_judgements = PARSED_RELEVANCE_JUDGEMENTS[subset][task]
+    for topic_name, results in parsed_run.items():
+        if topic_name not in parsed_relevance_judgements:
+            del only_judged_parsed_run[topic_name]
+        else:
+            judgements = parsed_relevance_judgements[topic_name]
+            for document_name in results.keys():
+                if document_name not in judgements:
+                    del only_judged_parsed_run[topic_name][document_name]
+    evaluator = EVALUATORS[subset][task]
+    evaluation = evaluator.evaluate(only_judged_parsed_run)
     ndcg = np.mean([
         measures['ndcg']
         for topic, measures

scripts/configuration.py

@@ -33,8 +33,10 @@ RELEVANCE_JUDGEMENTS = {
 }
 TASKS = list(RELEVANCE_JUDGEMENTS['test'].keys())
 PARSED_RELEVANCE_JUDGEMENTS = {}
+EVALUATORS = {}
 for subset, filenames in RELEVANCE_JUDGEMENTS.items():
     PARSED_RELEVANCE_JUDGEMENTS[subset] = {}
+    EVALUATORS[subset] = {}
     for task, filename in filenames.items():
         relevance_judgements_filename = os.path.join(
             os.path.dirname(__file__),
@@ -43,4 +45,5 @@ for subset, filenames in RELEVANCE_JUDGEMENTS.items():
         with open(relevance_judgements_filename, 'rt') as f:
             parsed_relevance_judgements = parse_qrel(f)
         evaluator = RelevanceEvaluator(parsed_relevance_judgements, {'ndcg'})
-        PARSED_RELEVANCE_JUDGEMENTS[subset][task] = evaluator
+        PARSED_RELEVANCE_JUDGEMENTS[subset][task] = parsed_relevance_judgements
+        EVALUATORS[subset][task] = evaluator