Add evaluation

main.py

@@ -1,3 +1,5 @@
+from collections import namedtuple
+from pprint import pprint
 from dotenv import load_dotenv
 # needs to happen as very first thing, otherwise HF ignores env vars
 load_dotenv()
@@ -5,8 +7,8 @@ load_dotenv()
 import os
 import pandas as pd
 
-from dataclasses import dataclass
-from typing import Dict, cast
+from dataclasses import dataclass, field
+from typing import Dict, cast, List
 from datasets import DatasetDict, load_dataset
 
 from src.readers.base_reader import Reader
@@ -24,10 +26,15 @@ from src.utils.preprocessing import context_to_reader_input
 from src.utils.timing import get_times, timeit
 
 
+ExperimentResult = namedtuple('ExperimentResult', ['correct', 'given'])
+
+
 @dataclass
 class Experiment:
     retriever: Retriever
     reader: Reader
+    lm: str
+    results: List[ExperimentResult] = field(default_factory=list)
 
 
 if __name__ == '__main__':
@@ -45,21 +52,25 @@ if __name__ == '__main__':
             retriever=FaissRetriever(
                 paragraphs,
                 FaissRetrieverOptions.dpr("./src/models/dpr.faiss")),
-            reader=DprReader()
+            reader=DprReader(),
+            lm="dpr"
         ),
         "faiss_longformer": Experiment(
             retriever=FaissRetriever(
                 paragraphs,
                 FaissRetrieverOptions.longformer("./src/models/longformer.faiss")),
-            reader=LongformerReader()
+            reader=LongformerReader(),
+            lm="longformer"
         ),
         "es_dpr": Experiment(
             retriever=ESRetriever(paragraphs),
-            reader=DprReader()
+            reader=DprReader(),
+            lm="dpr"
         ),
         "es_longformer": Experiment(
             retriever=ESRetriever(paragraphs),
-            reader=LongformerReader()
+            reader=LongformerReader(),
+            lm="longformer"
         ),
     }
 
@@ -69,6 +80,8 @@ if __name__ == '__main__':
             question = questions_test["question"][idx]
             answer = questions_test["answer"][idx]
 
+            # workaround so we can use the decorator with a dynamic name for
+            # time recording
             retrieve_timer = timeit(f"{experiment_name}.retrieve")
             t_retrieve = retrieve_timer(experiment.retriever.retrieve)
 
@@ -80,28 +93,38 @@ if __name__ == '__main__':
             scores, context = t_retrieve(question, 5)
             reader_input = context_to_reader_input(context)
 
-            # workaround so we can use the decorator with a dynamic name for
-            # time recording
-            answers = t_read(question, reader_input, 5)
-
-            # Calculate softmaxed scores for readable output
-            # sm = torch.nn.Softmax(dim=0)
-            # document_scores = sm(torch.Tensor(
-            #     [pred.relevance_score for pred in answers]))
-            # span_scores = sm(torch.Tensor(
-            #     [pred.span_score for pred in answers]))
+            # Requesting 1 answers results in us getting the best answer
+            given_answer = t_read(question, reader_input, 1)[0]
 
-            # print_answers(answers, scores, context)
+            # Save the results so we can evaluate laters
+            if experiment.lm == "longformer":
+                experiment.results.append(
+                    ExperimentResult(answer, given_answer[0]))
+            else:
+                experiment.results.append(
+                    ExperimentResult(answer, given_answer.text))
 
-            # TODO evaluation and storing of results
         print()
 
-    times = get_times()
+    if os.getenv("ENABLE_TIMING", "false").lower() == "true":
+        # Save times
+        times = get_times()
+        df = pd.DataFrame(times)
+        os.makedirs("./results/", exist_ok=True)
+        df.to_csv("./results/timings.csv")
 
-    df = pd.DataFrame(times)
-    os.makedirs("./results/", exist_ok=True)
-    df.to_csv("./results/timings.csv")
+    f1_results = pd.DataFrame(columns=experiments.keys())
+    em_results = pd.DataFrame(columns=experiments.keys())
+    for experiment_name, experiment in experiments.items():
+        em, f1 = zip(*list(map(
+            lambda r: evaluate(r.correct, r.given), experiment.results
+        )))
+        em_results[experiment_name] = em
+        f1_results[experiment_name] = f1
 
+    os.makedirs("./results/", exist_ok=True)
+    f1_results.to_csv("./results/f1_scores.csv")
+    em_results.to_csv("./results/em_scores.csv")
 
     # TODO evaluation and storing of results
 

results/em_scores.csv

@@ -0,0 +1,60 @@
+,faiss_dpr,faiss_longformer,es_dpr,es_longformer
+0,0,0,0,0
+1,0,0,0,0
+2,0,0,0,0
+3,0,0,0,0
+4,0,0,0,0
+5,0,0,0,0
+6,0,0,0,0
+7,0,0,0,0
+8,0,0,0,0
+9,0,0,0,0
+10,0,0,0,0
+11,0,0,0,0
+12,0,0,0,0
+13,0,0,0,0
+14,0,0,0,0
+15,0,0,0,0
+16,0,0,0,0
+17,0,0,0,0
+18,0,0,0,1
+19,0,0,0,0
+20,0,0,0,0
+21,0,0,0,0
+22,0,0,0,0
+23,0,0,0,0
+24,0,0,0,0
+25,0,0,0,0
+26,0,0,0,0
+27,0,0,0,0
+28,0,0,0,0
+29,0,0,0,0
+30,0,0,0,1
+31,0,0,0,0
+32,0,0,0,0
+33,0,0,0,0
+34,0,0,0,0
+35,0,0,0,0
+36,0,0,0,1
+37,0,0,0,1
+38,0,0,0,0
+39,0,0,0,0
+40,0,0,0,0
+41,0,0,0,0
+42,0,0,0,0
+43,0,0,0,0
+44,0,0,0,1
+45,0,0,0,0
+46,0,0,0,1
+47,0,0,0,0
+48,0,0,0,0
+49,0,0,0,0
+50,0,0,0,0
+51,0,0,0,0
+52,0,0,0,0
+53,0,0,0,0
+54,0,0,0,0
+55,0,0,0,0
+56,0,0,0,1
+57,0,0,0,0
+58,0,0,0,0

results/f1_scores.csv

@@ -0,0 +1,60 @@
+,faiss_dpr,faiss_longformer,es_dpr,es_longformer
+0,0.0,0.0,0.13008130081300812,0.7692307692307692
+1,0.0,0.0,0.0,0.5833333333333334
+2,0.0,0.0,0.3076923076923077,0.8421052631578948
+3,0.0,0.0,0.0,0.0
+4,0.25,0.0,0.25,0.88
+5,0.2222222222222222,0.08695652173913043,0.2222222222222222,0.5454545454545454
+6,0.0,0.0,0.0,0.10526315789473685
+7,0.0,0.0,0.0,0.14545454545454545
+8,0.0,0.0,0.0,0.7499999999999999
+9,0.1935483870967742,0.0,0.0,0.3913043478260869
+10,0.0,0.0,0.10526315789473685,0.0
+11,0.0,0.0,0.0,0.0
+12,0.07407407407407407,0.0,0.06896551724137931,0.0
+13,0.0,0.0,0.0,0.3076923076923077
+14,0.2222222222222222,0.0,0.29090909090909095,0.7142857142857143
+15,0.0,0.0,0.0,0.08695652173913043
+16,0.0,0.0,0.4347826086956522,0.30769230769230765
+17,0.0,0.0,0.5,0.0
+18,0.0,0.0,0.0,1.0
+19,0.0,0.0,0.07692307692307693,0.75
+20,0.0,0.046511627906976744,0.0,0.7333333333333334
+21,0.0,0.0,0.0,0.5806451612903226
+22,0.0,0.0,0.25,0.0
+23,0.0,0.0,0.7142857142857143,0.6153846153846153
+24,0.15384615384615383,0.0,0.15384615384615383,0.6666666666666666
+25,0.15384615384615383,0.0625,0.0909090909090909,0.0
+26,0.2285714285714286,0.05714285714285715,0.0,0.0
+27,0.19999999999999998,0.0,0.3636363636363636,0.0
+28,0.0,0.0,0.3076923076923077,0.16666666666666669
+29,0.5,0.0,0.07407407407407407,0.4
+30,0.11764705882352941,0.0,0.0,0.9375
+31,0.0,0.05405405405405406,0.12121212121212122,0.13953488372093023
+32,0.0,0.0,0.0,0.6
+33,0.0,0.0,0.0,0.3333333333333333
+34,0.07692307692307693,0.06896551724137931,0.07407407407407407,0.8
+35,0.0,0.0,0.0,0.049999999999999996
+36,0.0,0.0,0.0,1.0
+37,0.22222222222222224,0.0,0.0,0.7142857142857143
+38,0.058823529411764705,0.0,0.0,0.0
+39,0.33333333333333326,0.05128205128205129,0.33333333333333326,0.33333333333333326
+40,0.5882352941176471,0.0,0.0,0.0
+41,0.0,0.0,0.0909090909090909,0.0
+42,0.0,0.0,0.0,0.0
+43,0.0,0.0,0.0,0.0588235294117647
+44,0.0,0.0,0.19999999999999998,0.8888888888888888
+45,0.0,0.05714285714285714,0.13793103448275865,0.10256410256410256
+46,0.0,0.07142857142857142,0.0,0.8888888888888888
+47,0.19999999999999998,0.0,0.5714285714285714,0.9473684210526316
+48,0.0,0.0,0.0,0.0
+49,0.0,0.0,0.0,0.0
+50,0.13333333333333333,0.0,0.125,0.17391304347826086
+51,0.0,0.0,0.0,0.21052631578947367
+52,0.0,0.0,0.28571428571428575,0.0
+53,0.07692307692307691,0.06060606060606061,0.0,0.0
+54,0.0,0.11111111111111112,0.0,0.6153846153846153
+55,0.23809523809523808,0.0,0.0,0.19999999999999998
+56,0.0,0.0,0.0,1.0
+57,0.0,0.0,0.0,0.0
+58,0.0,0.0,0.0,0.13333333333333333

src/evaluation.py

@@ -74,4 +74,5 @@ def evaluate(answer: Any, prediction: Any):
         float: overall exact match
         float: overall F1-score
     """
+    print(prediction, answer)
     return exact_match(prediction, answer), f1(prediction, answer)

src/readers/dpr_reader.py

@@ -13,8 +13,7 @@ class DprReader(Reader):
         self._tokenizer = DPRReaderTokenizer.from_pretrained(
             "facebook/dpr-reader-single-nq-base")
         self._model = DPRReader.from_pretrained(
-            "facebook/dpr-reader-single-nq-base"
-        )
+            "facebook/dpr-reader-single-nq-base")
 
     def read(self,
              query: str,

src/readers/longformer_reader.py

@@ -24,7 +24,7 @@ class LongformerReader(Reader):
              num_answers=5) -> List[Tuple]:
         answers = []
 
-        for text in context['texts']:
+        for text in context['texts'][:num_answers]:
             encoding = self.tokenizer(query, text, return_tensors="pt")
             input_ids = encoding["input_ids"]
             attention_mask = encoding["attention_mask"]

src/retrievers/faiss_retriever.py

@@ -98,7 +98,8 @@ class FaissRetriever(Retriever):
     def _embed_question(self, q):
         match self.lm:
             case "dpr":
-                tok = self.q_tokenizer(q, return_tensors="pt", truncation=True)
+                tok = self.q_tokenizer(
+                    q, return_tensors="pt", truncation=True, padding=True)
                 return self.q_encoder(**tok)[0][0].numpy()
             case "longformer":
                 tok = self.q_tokenizer(q, return_tensors="pt")
@@ -110,7 +111,7 @@ class FaissRetriever(Retriever):
         match self.lm:
             case "dpr":
                 tok = self.ctx_tokenizer(
-                    p, return_tensors="pt", truncation=True)
+                    p, return_tensors="pt", truncation=True, padding=True)
                 enc = self.ctx_encoder(**tok)[0][0].numpy()
                 return {"embeddings": enc}
             case "longformer":

src/utils/preprocessing.py

@@ -17,7 +17,8 @@ def context_to_reader_input(result: Dict[str, List[str]]) \
     # Prepare result
     reader_result = {
         'titles': [],
-        'texts': []
+        'texts': [],
+        'scores': []
     }
 
     for n in range(num_entries):
@@ -31,6 +32,7 @@ def context_to_reader_input(result: Dict[str, List[str]]) \
 
         reader_result['titles'].append(title)
         reader_result['texts'].append(result['text'][n])
+        reader_result['scores'].append(result['text'][n])
 
     return reader_result