Merge branch 'main' of github.com:RamonMeffert/nlp-flashcard-project into main

results/em_analysis.ipynb

@@ -0,0 +1,457 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# F1 Scores"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "vscode": {
+     "languageId": "r"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Loading required package: ggplot2\n",
+      "\n",
+      "-- \u001b[1mAttaching packages\u001b[22m --------------------------------------- tidyverse 1.3.1 --\n",
+      "\n",
+      "\u001b[32mv\u001b[39m \u001b[34mtibble \u001b[39m 3.1.5     \u001b[32mv\u001b[39m \u001b[34mdplyr  \u001b[39m 1.0.7\n",
+      "\u001b[32mv\u001b[39m \u001b[34mtidyr  \u001b[39m 1.1.4     \u001b[32mv\u001b[39m \u001b[34mstringr\u001b[39m 1.4.0\n",
+      "\u001b[32mv\u001b[39m \u001b[34mpurrr  \u001b[39m 0.3.4     \u001b[32mv\u001b[39m \u001b[34mforcats\u001b[39m 0.5.1\n",
+      "\n",
+      "-- \u001b[1mConflicts\u001b[22m ------------------------------------------ tidyverse_conflicts() --\n",
+      "\u001b[31mx\u001b[39m \u001b[34mdplyr\u001b[39m::\u001b[32mfilter()\u001b[39m masks \u001b[34mstats\u001b[39m::filter()\n",
+      "\u001b[31mx\u001b[39m \u001b[34mdplyr\u001b[39m::\u001b[32mlag()\u001b[39m    masks \u001b[34mstats\u001b[39m::lag()\n",
+      "\n",
+      "Loading required package: mvtnorm\n",
+      "\n",
+      "Loading required package: survival\n",
+      "\n",
+      "Loading required package: TH.data\n",
+      "\n",
+      "Loading required package: MASS\n",
+      "\n",
+      "\n",
+      "Attaching package: 'MASS'\n",
+      "\n",
+      "\n",
+      "The following object is masked from 'package:dplyr':\n",
+      "\n",
+      "    select\n",
+      "\n",
+      "\n",
+      "\n",
+      "Attaching package: 'TH.data'\n",
+      "\n",
+      "\n",
+      "The following object is masked from 'package:MASS':\n",
+      "\n",
+      "    geyser\n",
+      "\n",
+      "\n",
+      "Loading required package: carData\n",
+      "\n",
+      "\n",
+      "Attaching package: 'car'\n",
+      "\n",
+      "\n",
+      "The following object is masked from 'package:dplyr':\n",
+      "\n",
+      "    recode\n",
+      "\n",
+      "\n",
+      "The following object is masked from 'package:purrr':\n",
+      "\n",
+      "    some\n",
+      "\n",
+      "\n",
+      "\n",
+      "Attaching package: 'rstatix'\n",
+      "\n",
+      "\n",
+      "The following object is masked from 'package:MASS':\n",
+      "\n",
+      "    select\n",
+      "\n",
+      "\n",
+      "The following object is masked from 'package:stats':\n",
+      "\n",
+      "    filter\n",
+      "\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "library(\"ggpubr\")\n",
+    "library(readr)\n",
+    "library(ggplot2)\n",
+    "library(tidyverse)\n",
+    "library(ARTool)\n",
+    "library(emmeans)\n",
+    "library(multcomp)\n",
+    "library(car)\n",
+    "library(rstatix)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "vscode": {
+     "languageId": "r"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "New names:\n",
+      "* `` -> ...1\n",
+      "\n",
+      "\u001b[1mRows: \u001b[22m\u001b[34m59\u001b[39m \u001b[1mColumns: \u001b[22m\u001b[34m5\u001b[39m\n",
+      "\u001b[36m--\u001b[39m \u001b[1mColumn specification\u001b[22m \u001b[36m--------------------------------------------------------\u001b[39m\n",
+      "\u001b[1mDelimiter:\u001b[22m \",\"\n",
+      "\u001b[32mdbl\u001b[39m (5): ...1, faiss_dpr, faiss_longformer, es_dpr, es_longformer\n",
+      "\n",
+      "\u001b[36mi\u001b[39m Use `spec()` to retrieve the full column specification for this data.\n",
+      "\u001b[36mi\u001b[39m Specify the column types or set `show_col_types = FALSE` to quiet this message.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<table class=\"dataframe\">\n",
+       "<caption>A tibble: 6 × 4</caption>\n",
+       "<thead>\n",
+       "\t<tr><th scope=col>question</th><th scope=col>retriever</th><th scope=col>reader</th><th scope=col>em</th></tr>\n",
+       "\t<tr><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;fct&gt;</th><th scope=col>&lt;fct&gt;</th><th scope=col>&lt;dbl&gt;</th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "\t<tr><td>0</td><td>faiss</td><td>dpr       </td><td>0</td></tr>\n",
+       "\t<tr><td>0</td><td>faiss</td><td>longformer</td><td>0</td></tr>\n",
+       "\t<tr><td>0</td><td>es   </td><td>dpr       </td><td>0</td></tr>\n",
+       "\t<tr><td>0</td><td>es   </td><td>longformer</td><td>0</td></tr>\n",
+       "\t<tr><td>1</td><td>faiss</td><td>dpr       </td><td>0</td></tr>\n",
+       "\t<tr><td>1</td><td>faiss</td><td>longformer</td><td>0</td></tr>\n",
+       "</tbody>\n",
+       "</table>\n"
+      ],
+      "text/latex": [
+       "A tibble: 6 × 4\n",
+       "\\begin{tabular}{llll}\n",
+       " question & retriever & reader & em\\\\\n",
+       " <dbl> & <fct> & <fct> & <dbl>\\\\\n",
+       "\\hline\n",
+       "\t 0 & faiss & dpr        & 0\\\\\n",
+       "\t 0 & faiss & longformer & 0\\\\\n",
+       "\t 0 & es    & dpr        & 0\\\\\n",
+       "\t 0 & es    & longformer & 0\\\\\n",
+       "\t 1 & faiss & dpr        & 0\\\\\n",
+       "\t 1 & faiss & longformer & 0\\\\\n",
+       "\\end{tabular}\n"
+      ],
+      "text/markdown": [
+       "\n",
+       "A tibble: 6 × 4\n",
+       "\n",
+       "| question &lt;dbl&gt; | retriever &lt;fct&gt; | reader &lt;fct&gt; | em &lt;dbl&gt; |\n",
+       "|---|---|---|---|\n",
+       "| 0 | faiss | dpr        | 0 |\n",
+       "| 0 | faiss | longformer | 0 |\n",
+       "| 0 | es    | dpr        | 0 |\n",
+       "| 0 | es    | longformer | 0 |\n",
+       "| 1 | faiss | dpr        | 0 |\n",
+       "| 1 | faiss | longformer | 0 |\n",
+       "\n"
+      ],
+      "text/plain": [
+       "  question retriever reader     em\n",
+       "1 0        faiss     dpr        0 \n",
+       "2 0        faiss     longformer 0 \n",
+       "3 0        es        dpr        0 \n",
+       "4 0        es        longformer 0 \n",
+       "5 1        faiss     dpr        0 \n",
+       "6 1        faiss     longformer 0 "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "em_scores <- read_csv(\"em_scores.csv\") %>%\n",
+    "    rename(question = `...1`) %>%\n",
+    "    pivot_longer(!question, names_to=c(\"retriever\", \"reader\"), names_sep=\"_\", values_to=\"em\")\n",
+    "\n",
+    "em_scores$retriever <- as.factor(em_scores$retriever)\n",
+    "em_scores$reader <- as.factor(em_scores$reader)\n",
+    "\n",
+    "head(em_scores)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "To test which tests we can use, we need to check for normality. For this, we use a Shapiro-Wilk test of normality. In this case, results with FAISS as retriever or DPR had reader had zero exact matches, thus making it impossible to compute the Shapiro-Wilk test of normality. Nonetheless, we know that a distribution with all-identical values is not normally distributed. As you can see in the results below, all other $p$-values are lower than 0.001, so we reject the null-hypothesis of normality and now know that none of the f1-scores are normally distributed."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {
+    "vscode": {
+     "languageId": "r"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table class=\"dataframe\">\n",
+       "<caption>A tibble: 1 × 3</caption>\n",
+       "<thead>\n",
+       "\t<tr><th scope=col>retriever</th><th scope=col>sw.stat</th><th scope=col>sw.p</th></tr>\n",
+       "\t<tr><th scope=col>&lt;fct&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "\t<tr><td>es</td><td>0.2503666</td><td>6.788451e-22</td></tr>\n",
+       "</tbody>\n",
+       "</table>\n"
+      ],
+      "text/latex": [
+       "A tibble: 1 × 3\n",
+       "\\begin{tabular}{lll}\n",
+       " retriever & sw.stat & sw.p\\\\\n",
+       " <fct> & <dbl> & <dbl>\\\\\n",
+       "\\hline\n",
+       "\t es & 0.2503666 & 6.788451e-22\\\\\n",
+       "\\end{tabular}\n"
+      ],
+      "text/markdown": [
+       "\n",
+       "A tibble: 1 × 3\n",
+       "\n",
+       "| retriever &lt;fct&gt; | sw.stat &lt;dbl&gt; | sw.p &lt;dbl&gt; |\n",
+       "|---|---|---|\n",
+       "| es | 0.2503666 | 6.788451e-22 |\n",
+       "\n"
+      ],
+      "text/plain": [
+       "  retriever sw.stat   sw.p        \n",
+       "1 es        0.2503666 6.788451e-22"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<table class=\"dataframe\">\n",
+       "<caption>A tibble: 1 × 3</caption>\n",
+       "<thead>\n",
+       "\t<tr><th scope=col>reader</th><th scope=col>sw.stat</th><th scope=col>sw.p</th></tr>\n",
+       "\t<tr><th scope=col>&lt;fct&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "\t<tr><td>longformer</td><td>0.2503666</td><td>6.788451e-22</td></tr>\n",
+       "</tbody>\n",
+       "</table>\n"
+      ],
+      "text/latex": [
+       "A tibble: 1 × 3\n",
+       "\\begin{tabular}{lll}\n",
+       " reader & sw.stat & sw.p\\\\\n",
+       " <fct> & <dbl> & <dbl>\\\\\n",
+       "\\hline\n",
+       "\t longformer & 0.2503666 & 6.788451e-22\\\\\n",
+       "\\end{tabular}\n"
+      ],
+      "text/markdown": [
+       "\n",
+       "A tibble: 1 × 3\n",
+       "\n",
+       "| reader &lt;fct&gt; | sw.stat &lt;dbl&gt; | sw.p &lt;dbl&gt; |\n",
+       "|---|---|---|\n",
+       "| longformer | 0.2503666 | 6.788451e-22 |\n",
+       "\n"
+      ],
+      "text/plain": [
+       "  reader     sw.stat   sw.p        \n",
+       "1 longformer 0.2503666 6.788451e-22"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "em_scores %>%\n",
+    "    select(!question) %>%\n",
+    "    group_by(retriever) %>%\n",
+    "    filter(sum(em) > 0) %>%\n",
+    "    summarise(sw.stat = shapiro.test(em)$statistic,\n",
+    "              sw.p = shapiro.test(em)$p)\n",
+    "em_scores %>%\n",
+    "    select(!question) %>%\n",
+    "    group_by(reader) %>%\n",
+    "    filter(sum(em) > 0) %>%\n",
+    "    summarise(sw.stat = shapiro.test(em)$statistic,\n",
+    "              sw.p = shapiro.test(em)$p)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Since our data is not normally distributed, we cannot use an ANOVA to compare our results. Therefore, we use an aligned-rank test, which is a non-parameteric version of a factorial repeated measures ANOVA."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "vscode": {
+     "languageId": "r"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table class=\"dataframe\">\n",
+       "<caption>A anova.art: 3 × 7</caption>\n",
+       "<thead>\n",
+       "\t<tr><th></th><th scope=col>Term</th><th scope=col>Df</th><th scope=col>Df.res</th><th scope=col>Sum Sq</th><th scope=col>Sum Sq.res</th><th scope=col>F value</th><th scope=col>Pr(&gt;F)</th></tr>\n",
+       "\t<tr><th></th><th scope=col>&lt;chr&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "\t<tr><th scope=row>retriever</th><td>retriever       </td><td>1</td><td>232</td><td>11564</td><td>263081</td><td>10.1978</td><td>0.001600976</td></tr>\n",
+       "\t<tr><th scope=row>reader</th><td>reader          </td><td>1</td><td>232</td><td>11564</td><td>263081</td><td>10.1978</td><td>0.001600976</td></tr>\n",
+       "\t<tr><th scope=row>retriever:reader</th><td>retriever:reader</td><td>1</td><td>232</td><td>11564</td><td>263081</td><td>10.1978</td><td>0.001600976</td></tr>\n",
+       "</tbody>\n",
+       "</table>\n"
+      ],
+      "text/latex": [
+       "A anova.art: 3 × 7\n",
+       "\\begin{tabular}{r|lllllll}\n",
+       "  & Term & Df & Df.res & Sum Sq & Sum Sq.res & F value & Pr(>F)\\\\\n",
+       "  & <chr> & <dbl> & <dbl> & <dbl> & <dbl> & <dbl> & <dbl>\\\\\n",
+       "\\hline\n",
+       "\tretriever & retriever        & 1 & 232 & 11564 & 263081 & 10.1978 & 0.001600976\\\\\n",
+       "\treader & reader           & 1 & 232 & 11564 & 263081 & 10.1978 & 0.001600976\\\\\n",
+       "\tretriever:reader & retriever:reader & 1 & 232 & 11564 & 263081 & 10.1978 & 0.001600976\\\\\n",
+       "\\end{tabular}\n"
+      ],
+      "text/markdown": [
+       "\n",
+       "A anova.art: 3 × 7\n",
+       "\n",
+       "| <!--/--> | Term &lt;chr&gt; | Df &lt;dbl&gt; | Df.res &lt;dbl&gt; | Sum Sq &lt;dbl&gt; | Sum Sq.res &lt;dbl&gt; | F value &lt;dbl&gt; | Pr(&gt;F) &lt;dbl&gt; |\n",
+       "|---|---|---|---|---|---|---|---|\n",
+       "| retriever | retriever        | 1 | 232 | 11564 | 263081 | 10.1978 | 0.001600976 |\n",
+       "| reader | reader           | 1 | 232 | 11564 | 263081 | 10.1978 | 0.001600976 |\n",
+       "| retriever:reader | retriever:reader | 1 | 232 | 11564 | 263081 | 10.1978 | 0.001600976 |\n",
+       "\n"
+      ],
+      "text/plain": [
+       "                 Term             Df Df.res Sum Sq Sum Sq.res F value\n",
+       "retriever        retriever        1  232    11564  263081     10.1978\n",
+       "reader           reader           1  232    11564  263081     10.1978\n",
+       "retriever:reader retriever:reader 1  232    11564  263081     10.1978\n",
+       "                 Pr(>F)     \n",
+       "retriever        0.001600976\n",
+       "reader           0.001600976\n",
+       "retriever:reader 0.001600976"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "NOTE: Results may be misleading due to involvement in interactions\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       " contrast   estimate   SE  df t.ratio p.value\n",
+       " es - faiss       14 4.38 232   3.193  0.0016\n",
+       "\n",
+       "Results are averaged over the levels of: reader "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "NOTE: Results may be misleading due to involvement in interactions\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       " contrast         estimate   SE  df t.ratio p.value\n",
+       " dpr - longformer      -14 4.38 232  -3.193  0.0016\n",
+       "\n",
+       "Results are averaged over the levels of: retriever "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "model.acc <- art(f1 ~ retriever * reader, data = em_scores)\n",
+    "anova(model.acc)\n",
+    "art.con(model.acc, ~ retriever)\n",
+    "art.con(model.acc, ~ reader)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "From these results, we can see that both the retriever and the reader have a significant effect on the F1 score ($F = 58.63$ and $F = 16.23$ respectively, $p < 0.0001$ for both). However, there is also an interaction between the retriever and reader ($F = 43.53$, $p < 0.0001$). The post-hoc analysis of contrasts shows that ElasticSearch performs better than FAISS ($p < 0.0001$) and Longformer performs better than DPR ($p = 0.0001$)."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "R",
+   "language": "R",
+   "name": "ir"
+  },
+  "language_info": {
+   "codemirror_mode": "r",
+   "file_extension": ".r",
+   "mimetype": "text/x-r-source",
+   "name": "R",
+   "pygments_lexer": "r",
+   "version": "4.1.2"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

results/f1_analysis.ipynb

@@ -0,0 +1,465 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# F1 Scores"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "vscode": {
+     "languageId": "r"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "library(\"ggpubr\")\n",
+    "library(readr)\n",
+    "library(ggplot2)\n",
+    "library(tidyverse)\n",
+    "library(ARTool)\n",
+    "library(emmeans)\n",
+    "library(multcomp)\n",
+    "library(car)\n",
+    "library(rstatix)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {
+    "vscode": {
+     "languageId": "r"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "New names:\n",
+      "* `` -> ...1\n",
+      "\n",
+      "\u001b[1mRows: \u001b[22m\u001b[34m59\u001b[39m \u001b[1mColumns: \u001b[22m\u001b[34m5\u001b[39m\n",
+      "\u001b[36m--\u001b[39m \u001b[1mColumn specification\u001b[22m \u001b[36m--------------------------------------------------------\u001b[39m\n",
+      "\u001b[1mDelimiter:\u001b[22m \",\"\n",
+      "\u001b[32mdbl\u001b[39m (5): ...1, faiss_dpr, faiss_longformer, es_dpr, es_longformer\n",
+      "\n",
+      "\u001b[36mi\u001b[39m Use `spec()` to retrieve the full column specification for this data.\n",
+      "\u001b[36mi\u001b[39m Specify the column types or set `show_col_types = FALSE` to quiet this message.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<table class=\"dataframe\">\n",
+       "<caption>A tibble: 6 × 4</caption>\n",
+       "<thead>\n",
+       "\t<tr><th scope=col>question</th><th scope=col>retriever</th><th scope=col>reader</th><th scope=col>f1</th></tr>\n",
+       "\t<tr><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;fct&gt;</th><th scope=col>&lt;fct&gt;</th><th scope=col>&lt;dbl&gt;</th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "\t<tr><td>0</td><td>faiss</td><td>dpr       </td><td>0.0000000</td></tr>\n",
+       "\t<tr><td>0</td><td>faiss</td><td>longformer</td><td>0.0000000</td></tr>\n",
+       "\t<tr><td>0</td><td>es   </td><td>dpr       </td><td>0.1300813</td></tr>\n",
+       "\t<tr><td>0</td><td>es   </td><td>longformer</td><td>0.7692308</td></tr>\n",
+       "\t<tr><td>1</td><td>faiss</td><td>dpr       </td><td>0.0000000</td></tr>\n",
+       "\t<tr><td>1</td><td>faiss</td><td>longformer</td><td>0.0000000</td></tr>\n",
+       "</tbody>\n",
+       "</table>\n"
+      ],
+      "text/latex": [
+       "A tibble: 6 × 4\n",
+       "\\begin{tabular}{llll}\n",
+       " question & retriever & reader & f1\\\\\n",
+       " <dbl> & <fct> & <fct> & <dbl>\\\\\n",
+       "\\hline\n",
+       "\t 0 & faiss & dpr        & 0.0000000\\\\\n",
+       "\t 0 & faiss & longformer & 0.0000000\\\\\n",
+       "\t 0 & es    & dpr        & 0.1300813\\\\\n",
+       "\t 0 & es    & longformer & 0.7692308\\\\\n",
+       "\t 1 & faiss & dpr        & 0.0000000\\\\\n",
+       "\t 1 & faiss & longformer & 0.0000000\\\\\n",
+       "\\end{tabular}\n"
+      ],
+      "text/markdown": [
+       "\n",
+       "A tibble: 6 × 4\n",
+       "\n",
+       "| question &lt;dbl&gt; | retriever &lt;fct&gt; | reader &lt;fct&gt; | f1 &lt;dbl&gt; |\n",
+       "|---|---|---|---|\n",
+       "| 0 | faiss | dpr        | 0.0000000 |\n",
+       "| 0 | faiss | longformer | 0.0000000 |\n",
+       "| 0 | es    | dpr        | 0.1300813 |\n",
+       "| 0 | es    | longformer | 0.7692308 |\n",
+       "| 1 | faiss | dpr        | 0.0000000 |\n",
+       "| 1 | faiss | longformer | 0.0000000 |\n",
+       "\n"
+      ],
+      "text/plain": [
+       "  question retriever reader     f1       \n",
+       "1 0        faiss     dpr        0.0000000\n",
+       "2 0        faiss     longformer 0.0000000\n",
+       "3 0        es        dpr        0.1300813\n",
+       "4 0        es        longformer 0.7692308\n",
+       "5 1        faiss     dpr        0.0000000\n",
+       "6 1        faiss     longformer 0.0000000"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "f1_scores <- read_csv(\"f1_scores.csv\") %>%\n",
+    "    rename(question = `...1`) %>%\n",
+    "    pivot_longer(!question, names_to=c(\"retriever\", \"reader\"), names_sep=\"_\", values_to=\"f1\")\n",
+    "\n",
+    "f1_scores$retriever = as.factor(f1_scores$retriever)\n",
+    "f1_scores$reader = as.factor(f1_scores$reader)\n",
+    "\n",
+    "head(f1_scores)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "To test which tests we can use, we need to check for normality. For this, we use a Shapiro-Wilk test of normality. As you can see in the results below, all $p$-values are lower than 0.001, so we reject the null-hypothesis of normality and now know that none of the f1-scores are normally distributed."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {
+    "vscode": {
+     "languageId": "r"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table class=\"dataframe\">\n",
+       "<caption>A tibble: 1 × 3</caption>\n",
+       "<thead>\n",
+       "\t<tr><th scope=col>variable</th><th scope=col>statistic</th><th scope=col>p</th></tr>\n",
+       "\t<tr><th scope=col>&lt;chr&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "\t<tr><td>f1</td><td>0.5086706</td><td>3.999447e-18</td></tr>\n",
+       "</tbody>\n",
+       "</table>\n"
+      ],
+      "text/latex": [
+       "A tibble: 1 × 3\n",
+       "\\begin{tabular}{lll}\n",
+       " variable & statistic & p\\\\\n",
+       " <chr> & <dbl> & <dbl>\\\\\n",
+       "\\hline\n",
+       "\t f1 & 0.5086706 & 3.999447e-18\\\\\n",
+       "\\end{tabular}\n"
+      ],
+      "text/markdown": [
+       "\n",
+       "A tibble: 1 × 3\n",
+       "\n",
+       "| variable &lt;chr&gt; | statistic &lt;dbl&gt; | p &lt;dbl&gt; |\n",
+       "|---|---|---|\n",
+       "| f1 | 0.5086706 | 3.999447e-18 |\n",
+       "\n"
+      ],
+      "text/plain": [
+       "  variable statistic p           \n",
+       "1 f1       0.5086706 3.999447e-18"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<table class=\"dataframe\">\n",
+       "<caption>A tibble: 1 × 3</caption>\n",
+       "<thead>\n",
+       "\t<tr><th scope=col>variable</th><th scope=col>statistic</th><th scope=col>p</th></tr>\n",
+       "\t<tr><th scope=col>&lt;chr&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "\t<tr><td>f1</td><td>0.7704567</td><td>2.671656e-12</td></tr>\n",
+       "</tbody>\n",
+       "</table>\n"
+      ],
+      "text/latex": [
+       "A tibble: 1 × 3\n",
+       "\\begin{tabular}{lll}\n",
+       " variable & statistic & p\\\\\n",
+       " <chr> & <dbl> & <dbl>\\\\\n",
+       "\\hline\n",
+       "\t f1 & 0.7704567 & 2.671656e-12\\\\\n",
+       "\\end{tabular}\n"
+      ],
+      "text/markdown": [
+       "\n",
+       "A tibble: 1 × 3\n",
+       "\n",
+       "| variable &lt;chr&gt; | statistic &lt;dbl&gt; | p &lt;dbl&gt; |\n",
+       "|---|---|---|\n",
+       "| f1 | 0.7704567 | 2.671656e-12 |\n",
+       "\n"
+      ],
+      "text/plain": [
+       "  variable statistic p           \n",
+       "1 f1       0.7704567 2.671656e-12"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<table class=\"dataframe\">\n",
+       "<caption>A tibble: 1 × 3</caption>\n",
+       "<thead>\n",
+       "\t<tr><th scope=col>variable</th><th scope=col>statistic</th><th scope=col>p</th></tr>\n",
+       "\t<tr><th scope=col>&lt;chr&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "\t<tr><td>f1</td><td>0.6741031</td><td>7.912632e-15</td></tr>\n",
+       "</tbody>\n",
+       "</table>\n"
+      ],
+      "text/latex": [
+       "A tibble: 1 × 3\n",
+       "\\begin{tabular}{lll}\n",
+       " variable & statistic & p\\\\\n",
+       " <chr> & <dbl> & <dbl>\\\\\n",
+       "\\hline\n",
+       "\t f1 & 0.6741031 & 7.912632e-15\\\\\n",
+       "\\end{tabular}\n"
+      ],
+      "text/markdown": [
+       "\n",
+       "A tibble: 1 × 3\n",
+       "\n",
+       "| variable &lt;chr&gt; | statistic &lt;dbl&gt; | p &lt;dbl&gt; |\n",
+       "|---|---|---|\n",
+       "| f1 | 0.6741031 | 7.912632e-15 |\n",
+       "\n"
+      ],
+      "text/plain": [
+       "  variable statistic p           \n",
+       "1 f1       0.6741031 7.912632e-15"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<table class=\"dataframe\">\n",
+       "<caption>A tibble: 1 × 3</caption>\n",
+       "<thead>\n",
+       "\t<tr><th scope=col>variable</th><th scope=col>statistic</th><th scope=col>p</th></tr>\n",
+       "\t<tr><th scope=col>&lt;chr&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "\t<tr><td>f1</td><td>0.6558935</td><td>3.037616e-15</td></tr>\n",
+       "</tbody>\n",
+       "</table>\n"
+      ],
+      "text/latex": [
+       "A tibble: 1 × 3\n",
+       "\\begin{tabular}{lll}\n",
+       " variable & statistic & p\\\\\n",
+       " <chr> & <dbl> & <dbl>\\\\\n",
+       "\\hline\n",
+       "\t f1 & 0.6558935 & 3.037616e-15\\\\\n",
+       "\\end{tabular}\n"
+      ],
+      "text/markdown": [
+       "\n",
+       "A tibble: 1 × 3\n",
+       "\n",
+       "| variable &lt;chr&gt; | statistic &lt;dbl&gt; | p &lt;dbl&gt; |\n",
+       "|---|---|---|\n",
+       "| f1 | 0.6558935 | 3.037616e-15 |\n",
+       "\n"
+      ],
+      "text/plain": [
+       "  variable statistic p           \n",
+       "1 f1       0.6558935 3.037616e-15"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "f1_scores %>%\n",
+    "    filter(retriever == \"faiss\") %>%\n",
+    "    shapiro_test(f1)\n",
+    "\n",
+    "f1_scores %>%\n",
+    "    filter(retriever == \"es\") %>%\n",
+    "    shapiro_test(f1)\n",
+    "\n",
+    "f1_scores %>%\n",
+    "    filter(reader == \"dpr\") %>%\n",
+    "    shapiro_test(f1)\n",
+    "\n",
+    "f1_scores %>%\n",
+    "    filter(reader == \"longformer\") %>%\n",
+    "    shapiro_test(f1)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Since our data is not normally distributed, we cannot use an ANOVA to compare our results. Therefore, we use an aligned-rank test, which is a non-parameteric version of a factorial repeated measures ANOVA."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {
+    "vscode": {
+     "languageId": "r"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table class=\"dataframe\">\n",
+       "<caption>A anova.art: 3 × 7</caption>\n",
+       "<thead>\n",
+       "\t<tr><th></th><th scope=col>Term</th><th scope=col>Df</th><th scope=col>Df.res</th><th scope=col>Sum Sq</th><th scope=col>Sum Sq.res</th><th scope=col>F value</th><th scope=col>Pr(&gt;F)</th></tr>\n",
+       "\t<tr><th></th><th scope=col>&lt;chr&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "\t<tr><th scope=row>retriever</th><td>retriever       </td><td>1</td><td>232</td><td>200452.90</td><td>793168.0</td><td>58.63206</td><td>5.105423e-13</td></tr>\n",
+       "\t<tr><th scope=row>reader</th><td>reader          </td><td>1</td><td>232</td><td> 66045.36</td><td>944311.6</td><td>16.22613</td><td>7.620176e-05</td></tr>\n",
+       "\t<tr><th scope=row>retriever:reader</th><td>retriever:reader</td><td>1</td><td>232</td><td>158290.44</td><td>843714.0</td><td>43.52587</td><td>2.804257e-10</td></tr>\n",
+       "</tbody>\n",
+       "</table>\n"
+      ],
+      "text/latex": [
+       "A anova.art: 3 × 7\n",
+       "\\begin{tabular}{r|lllllll}\n",
+       "  & Term & Df & Df.res & Sum Sq & Sum Sq.res & F value & Pr(>F)\\\\\n",
+       "  & <chr> & <dbl> & <dbl> & <dbl> & <dbl> & <dbl> & <dbl>\\\\\n",
+       "\\hline\n",
+       "\tretriever & retriever        & 1 & 232 & 200452.90 & 793168.0 & 58.63206 & 5.105423e-13\\\\\n",
+       "\treader & reader           & 1 & 232 &  66045.36 & 944311.6 & 16.22613 & 7.620176e-05\\\\\n",
+       "\tretriever:reader & retriever:reader & 1 & 232 & 158290.44 & 843714.0 & 43.52587 & 2.804257e-10\\\\\n",
+       "\\end{tabular}\n"
+      ],
+      "text/markdown": [
+       "\n",
+       "A anova.art: 3 × 7\n",
+       "\n",
+       "| <!--/--> | Term &lt;chr&gt; | Df &lt;dbl&gt; | Df.res &lt;dbl&gt; | Sum Sq &lt;dbl&gt; | Sum Sq.res &lt;dbl&gt; | F value &lt;dbl&gt; | Pr(&gt;F) &lt;dbl&gt; |\n",
+       "|---|---|---|---|---|---|---|---|\n",
+       "| retriever | retriever        | 1 | 232 | 200452.90 | 793168.0 | 58.63206 | 5.105423e-13 |\n",
+       "| reader | reader           | 1 | 232 |  66045.36 | 944311.6 | 16.22613 | 7.620176e-05 |\n",
+       "| retriever:reader | retriever:reader | 1 | 232 | 158290.44 | 843714.0 | 43.52587 | 2.804257e-10 |\n",
+       "\n"
+      ],
+      "text/plain": [
+       "                 Term             Df Df.res Sum Sq    Sum Sq.res F value \n",
+       "retriever        retriever        1  232    200452.90 793168.0   58.63206\n",
+       "reader           reader           1  232     66045.36 944311.6   16.22613\n",
+       "retriever:reader retriever:reader 1  232    158290.44 843714.0   43.52587\n",
+       "                 Pr(>F)      \n",
+       "retriever        5.105423e-13\n",
+       "reader           7.620176e-05\n",
+       "retriever:reader 2.804257e-10"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "NOTE: Results may be misleading due to involvement in interactions\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       " contrast   estimate   SE  df t.ratio p.value\n",
+       " es - faiss     58.3 7.61 232   7.657  <.0001\n",
+       "\n",
+       "Results are averaged over the levels of: reader "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "NOTE: Results may be misleading due to involvement in interactions\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       " contrast         estimate   SE  df t.ratio p.value\n",
+       " dpr - longformer    -33.5 8.31 232  -4.028  0.0001\n",
+       "\n",
+       "Results are averaged over the levels of: retriever "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "model.acc <- art(f1 ~ retriever * reader, data = f1_scores)\n",
+    "anova(model.acc)\n",
+    "art.con(model.acc, ~ retriever)\n",
+    "art.con(model.acc, ~ reader)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "From these results, we can see that both the retriever and the reader have a significant effect on the F1 score ($F = 58.63$ and $F = 16.23$ respectively, $p < 0.0001$ for both). However, there is also an interaction between the retriever and reader ($F = 43.53$, $p < 0.0001$). The post-hoc analysis of contrasts shows that ElasticSearch performs better than FAISS ($p < 0.0001$) and Longformer performs better than DPR ($p = 0.0001$)."
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "R",
+   "language": "R",
+   "name": "ir"
+  },
+  "language_info": {
+   "codemirror_mode": "r",
+   "file_extension": ".r",
+   "mimetype": "text/x-r-source",
+   "name": "R",
+   "pygments_lexer": "r",
+   "version": "4.1.2"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

results/timings_analysis.ipynb

@@ -0,0 +1,635 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Timings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "vscode": {
+     "languageId": "r"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Loading required package: ggplot2\n",
+      "\n",
+      "-- \u001b[1mAttaching packages\u001b[22m --------------------------------------- tidyverse 1.3.1 --\n",
+      "\n",
+      "\u001b[32mv\u001b[39m \u001b[34mtibble \u001b[39m 3.1.5     \u001b[32mv\u001b[39m \u001b[34mdplyr  \u001b[39m 1.0.7\n",
+      "\u001b[32mv\u001b[39m \u001b[34mtidyr  \u001b[39m 1.1.4     \u001b[32mv\u001b[39m \u001b[34mstringr\u001b[39m 1.4.0\n",
+      "\u001b[32mv\u001b[39m \u001b[34mpurrr  \u001b[39m 0.3.4     \u001b[32mv\u001b[39m \u001b[34mforcats\u001b[39m 0.5.1\n",
+      "\n",
+      "-- \u001b[1mConflicts\u001b[22m ------------------------------------------ tidyverse_conflicts() --\n",
+      "\u001b[31mx\u001b[39m \u001b[34mdplyr\u001b[39m::\u001b[32mfilter()\u001b[39m masks \u001b[34mstats\u001b[39m::filter()\n",
+      "\u001b[31mx\u001b[39m \u001b[34mdplyr\u001b[39m::\u001b[32mlag()\u001b[39m    masks \u001b[34mstats\u001b[39m::lag()\n",
+      "\n",
+      "Loading required package: mvtnorm\n",
+      "\n",
+      "Loading required package: survival\n",
+      "\n",
+      "Loading required package: TH.data\n",
+      "\n",
+      "Loading required package: MASS\n",
+      "\n",
+      "\n",
+      "Attaching package: 'MASS'\n",
+      "\n",
+      "\n",
+      "The following object is masked from 'package:dplyr':\n",
+      "\n",
+      "    select\n",
+      "\n",
+      "\n",
+      "\n",
+      "Attaching package: 'TH.data'\n",
+      "\n",
+      "\n",
+      "The following object is masked from 'package:MASS':\n",
+      "\n",
+      "    geyser\n",
+      "\n",
+      "\n",
+      "Loading required package: carData\n",
+      "\n",
+      "\n",
+      "Attaching package: 'car'\n",
+      "\n",
+      "\n",
+      "The following object is masked from 'package:dplyr':\n",
+      "\n",
+      "    recode\n",
+      "\n",
+      "\n",
+      "The following object is masked from 'package:purrr':\n",
+      "\n",
+      "    some\n",
+      "\n",
+      "\n",
+      "\n",
+      "Attaching package: 'rstatix'\n",
+      "\n",
+      "\n",
+      "The following object is masked from 'package:MASS':\n",
+      "\n",
+      "    select\n",
+      "\n",
+      "\n",
+      "The following object is masked from 'package:stats':\n",
+      "\n",
+      "    filter\n",
+      "\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "library(\"ggpubr\")\n",
+    "library(readr)\n",
+    "library(ggplot2)\n",
+    "library(tidyverse)\n",
+    "library(ARTool)\n",
+    "library(emmeans)\n",
+    "library(multcomp)\n",
+    "library(car)\n",
+    "library(rstatix)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "vscode": {
+     "languageId": "r"
+    }
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "New names:\n",
+      "* `` -> ...1\n",
+      "\n",
+      "\u001b[1mRows: \u001b[22m\u001b[34m59\u001b[39m \u001b[1mColumns: \u001b[22m\u001b[34m9\u001b[39m\n",
+      "\u001b[36m--\u001b[39m \u001b[1mColumn specification\u001b[22m \u001b[36m--------------------------------------------------------\u001b[39m\n",
+      "\u001b[1mDelimiter:\u001b[22m \",\"\n",
+      "\u001b[32mdbl\u001b[39m (9): ...1, faiss_dpr.retrieve, faiss_dpr.read, faiss_longformer.retrieve...\n",
+      "\n",
+      "\u001b[36mi\u001b[39m Use `spec()` to retrieve the full column specification for this data.\n",
+      "\u001b[36mi\u001b[39m Specify the column types or set `show_col_types = FALSE` to quiet this message.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<table class=\"dataframe\">\n",
+       "<caption>A tibble: 6 × 5</caption>\n",
+       "<thead>\n",
+       "\t<tr><th scope=col>question</th><th scope=col>retriever</th><th scope=col>reader</th><th scope=col>method</th><th scope=col>time</th></tr>\n",
+       "\t<tr><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;fct&gt;</th><th scope=col>&lt;fct&gt;</th><th scope=col>&lt;fct&gt;</th><th scope=col>&lt;dbl&gt;</th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "\t<tr><td>0</td><td>faiss</td><td>dpr       </td><td>retrieve</td><td>0.30384302</td></tr>\n",
+       "\t<tr><td>0</td><td>faiss</td><td>dpr       </td><td>read    </td><td>4.56640005</td></tr>\n",
+       "\t<tr><td>0</td><td>faiss</td><td>longformer</td><td>retrieve</td><td>0.92279482</td></tr>\n",
+       "\t<tr><td>0</td><td>faiss</td><td>longformer</td><td>read    </td><td>5.76836824</td></tr>\n",
+       "\t<tr><td>0</td><td>es   </td><td>dpr       </td><td>retrieve</td><td>0.01930094</td></tr>\n",
+       "\t<tr><td>0</td><td>es   </td><td>dpr       </td><td>read    </td><td>2.74536490</td></tr>\n",
+       "</tbody>\n",
+       "</table>\n"
+      ],
+      "text/latex": [
+       "A tibble: 6 × 5\n",
+       "\\begin{tabular}{lllll}\n",
+       " question & retriever & reader & method & time\\\\\n",
+       " <dbl> & <fct> & <fct> & <fct> & <dbl>\\\\\n",
+       "\\hline\n",
+       "\t 0 & faiss & dpr        & retrieve & 0.30384302\\\\\n",
+       "\t 0 & faiss & dpr        & read     & 4.56640005\\\\\n",
+       "\t 0 & faiss & longformer & retrieve & 0.92279482\\\\\n",
+       "\t 0 & faiss & longformer & read     & 5.76836824\\\\\n",
+       "\t 0 & es    & dpr        & retrieve & 0.01930094\\\\\n",
+       "\t 0 & es    & dpr        & read     & 2.74536490\\\\\n",
+       "\\end{tabular}\n"
+      ],
+      "text/markdown": [
+       "\n",
+       "A tibble: 6 × 5\n",
+       "\n",
+       "| question &lt;dbl&gt; | retriever &lt;fct&gt; | reader &lt;fct&gt; | method &lt;fct&gt; | time &lt;dbl&gt; |\n",
+       "|---|---|---|---|---|\n",
+       "| 0 | faiss | dpr        | retrieve | 0.30384302 |\n",
+       "| 0 | faiss | dpr        | read     | 4.56640005 |\n",
+       "| 0 | faiss | longformer | retrieve | 0.92279482 |\n",
+       "| 0 | faiss | longformer | read     | 5.76836824 |\n",
+       "| 0 | es    | dpr        | retrieve | 0.01930094 |\n",
+       "| 0 | es    | dpr        | read     | 2.74536490 |\n",
+       "\n"
+      ],
+      "text/plain": [
+       "  question retriever reader     method   time      \n",
+       "1 0        faiss     dpr        retrieve 0.30384302\n",
+       "2 0        faiss     dpr        read     4.56640005\n",
+       "3 0        faiss     longformer retrieve 0.92279482\n",
+       "4 0        faiss     longformer read     5.76836824\n",
+       "5 0        es        dpr        retrieve 0.01930094\n",
+       "6 0        es        dpr        read     2.74536490"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "timings <- read_csv(\"timings.csv\") %>%\n",
+    "    rename(question = `...1`) %>%\n",
+    "    pivot_longer(!question, names_to=c(\"retriever\", \"reader\", \"method\"), names_sep=\"[._]\", values_to=\"time\")\n",
+    "\n",
+    "timings$retriever <- as.factor(timings$retriever)\n",
+    "timings$reader    <- as.factor(timings$reader)\n",
+    "timings$method    <- as.factor(timings$method)\n",
+    "\n",
+    "head(timings)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "vscode": {
+     "languageId": "r"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "timings_read <- filter(timings, method == \"read\") %>%\n",
+    "    select(!method)\n",
+    "timings_retrieve <- filter(timings, method == \"retrieve\") %>%\n",
+    "    select(!method)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "To test which tests we can use, we need to check for normality. For this, we use a Shapiro-Wilk test of normality. As you can see in the results below, all $p$-values are lower than 0.001, so we reject the null-hypothesis of normality and now know that none of the f1-scores are normally distributed."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "vscode": {
+     "languageId": "r"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table class=\"dataframe\">\n",
+       "<caption>A tibble: 2 × 3</caption>\n",
+       "<thead>\n",
+       "\t<tr><th scope=col>retriever</th><th scope=col>sw.stat</th><th scope=col>sw.p</th></tr>\n",
+       "\t<tr><th scope=col>&lt;fct&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "\t<tr><td>es   </td><td>0.7534261</td><td>1.667341e-18</td></tr>\n",
+       "\t<tr><td>faiss</td><td>0.7585727</td><td>2.563192e-18</td></tr>\n",
+       "</tbody>\n",
+       "</table>\n"
+      ],
+      "text/latex": [
+       "A tibble: 2 × 3\n",
+       "\\begin{tabular}{lll}\n",
+       " retriever & sw.stat & sw.p\\\\\n",
+       " <fct> & <dbl> & <dbl>\\\\\n",
+       "\\hline\n",
+       "\t es    & 0.7534261 & 1.667341e-18\\\\\n",
+       "\t faiss & 0.7585727 & 2.563192e-18\\\\\n",
+       "\\end{tabular}\n"
+      ],
+      "text/markdown": [
+       "\n",
+       "A tibble: 2 × 3\n",
+       "\n",
+       "| retriever &lt;fct&gt; | sw.stat &lt;dbl&gt; | sw.p &lt;dbl&gt; |\n",
+       "|---|---|---|\n",
+       "| es    | 0.7534261 | 1.667341e-18 |\n",
+       "| faiss | 0.7585727 | 2.563192e-18 |\n",
+       "\n"
+      ],
+      "text/plain": [
+       "  retriever sw.stat   sw.p        \n",
+       "1 es        0.7534261 1.667341e-18\n",
+       "2 faiss     0.7585727 2.563192e-18"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<table class=\"dataframe\">\n",
+       "<caption>A tibble: 2 × 3</caption>\n",
+       "<thead>\n",
+       "\t<tr><th scope=col>reader</th><th scope=col>sw.stat</th><th scope=col>sw.p</th></tr>\n",
+       "\t<tr><th scope=col>&lt;fct&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "\t<tr><td>dpr       </td><td>0.7639005</td><td>4.029344e-18</td></tr>\n",
+       "\t<tr><td>longformer</td><td>0.8116362</td><td>3.381683e-16</td></tr>\n",
+       "</tbody>\n",
+       "</table>\n"
+      ],
+      "text/latex": [
+       "A tibble: 2 × 3\n",
+       "\\begin{tabular}{lll}\n",
+       " reader & sw.stat & sw.p\\\\\n",
+       " <fct> & <dbl> & <dbl>\\\\\n",
+       "\\hline\n",
+       "\t dpr        & 0.7639005 & 4.029344e-18\\\\\n",
+       "\t longformer & 0.8116362 & 3.381683e-16\\\\\n",
+       "\\end{tabular}\n"
+      ],
+      "text/markdown": [
+       "\n",
+       "A tibble: 2 × 3\n",
+       "\n",
+       "| reader &lt;fct&gt; | sw.stat &lt;dbl&gt; | sw.p &lt;dbl&gt; |\n",
+       "|---|---|---|\n",
+       "| dpr        | 0.7639005 | 4.029344e-18 |\n",
+       "| longformer | 0.8116362 | 3.381683e-16 |\n",
+       "\n"
+      ],
+      "text/plain": [
+       "  reader     sw.stat   sw.p        \n",
+       "1 dpr        0.7639005 4.029344e-18\n",
+       "2 longformer 0.8116362 3.381683e-16"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<table class=\"dataframe\">\n",
+       "<caption>A tibble: 2 × 3</caption>\n",
+       "<thead>\n",
+       "\t<tr><th scope=col>method</th><th scope=col>sw.stat</th><th scope=col>sw.p</th></tr>\n",
+       "\t<tr><th scope=col>&lt;fct&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "\t<tr><td>read    </td><td>0.8838182</td><td>1.779766e-12</td></tr>\n",
+       "\t<tr><td>retrieve</td><td>0.6237773</td><td>1.838892e-22</td></tr>\n",
+       "</tbody>\n",
+       "</table>\n"
+      ],
+      "text/latex": [
+       "A tibble: 2 × 3\n",
+       "\\begin{tabular}{lll}\n",
+       " method & sw.stat & sw.p\\\\\n",
+       " <fct> & <dbl> & <dbl>\\\\\n",
+       "\\hline\n",
+       "\t read     & 0.8838182 & 1.779766e-12\\\\\n",
+       "\t retrieve & 0.6237773 & 1.838892e-22\\\\\n",
+       "\\end{tabular}\n"
+      ],
+      "text/markdown": [
+       "\n",
+       "A tibble: 2 × 3\n",
+       "\n",
+       "| method &lt;fct&gt; | sw.stat &lt;dbl&gt; | sw.p &lt;dbl&gt; |\n",
+       "|---|---|---|\n",
+       "| read     | 0.8838182 | 1.779766e-12 |\n",
+       "| retrieve | 0.6237773 | 1.838892e-22 |\n",
+       "\n"
+      ],
+      "text/plain": [
+       "  method   sw.stat   sw.p        \n",
+       "1 read     0.8838182 1.779766e-12\n",
+       "2 retrieve 0.6237773 1.838892e-22"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "timings %>%\n",
+    "    group_by(retriever) %>%\n",
+    "    summarise(sw.stat = shapiro.test(time)$statistic,\n",
+    "              sw.p = shapiro.test(time)$p)\n",
+    "\n",
+    "timings %>%\n",
+    "    group_by(reader) %>%\n",
+    "    summarise(sw.stat = shapiro.test(time)$statistic,\n",
+    "              sw.p = shapiro.test(time)$p)\n",
+    "\n",
+    "timings %>%\n",
+    "    group_by(method) %>%\n",
+    "    summarise(sw.stat = shapiro.test(time)$statistic,\n",
+    "              sw.p = shapiro.test(time)$p)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Since our data is not normally distributed, we cannot use an ANOVA to compare our results. Therefore, we use an aligned-rank test, which is a non-parameteric version of a factorial repeated measures ANOVA."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "vscode": {
+     "languageId": "r"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table class=\"dataframe\">\n",
+       "<caption>A anova.art: 3 × 7</caption>\n",
+       "<thead>\n",
+       "\t<tr><th></th><th scope=col>Term</th><th scope=col>Df</th><th scope=col>Df.res</th><th scope=col>Sum Sq</th><th scope=col>Sum Sq.res</th><th scope=col>F value</th><th scope=col>Pr(&gt;F)</th></tr>\n",
+       "\t<tr><th></th><th scope=col>&lt;chr&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "\t<tr><th scope=row>retriever</th><td>retriever       </td><td>1</td><td>232</td><td> 41088.97</td><td>1037631.8</td><td>  9.18692</td><td>2.714084e-03</td></tr>\n",
+       "\t<tr><th scope=row>reader</th><td>reader          </td><td>1</td><td>232</td><td>790427.81</td><td> 301414.1</td><td>608.39633</td><td>8.802730e-67</td></tr>\n",
+       "\t<tr><th scope=row>retriever:reader</th><td>retriever:reader</td><td>1</td><td>232</td><td>101903.46</td><td> 983331.4</td><td> 24.04235</td><td>1.771995e-06</td></tr>\n",
+       "</tbody>\n",
+       "</table>\n"
+      ],
+      "text/latex": [
+       "A anova.art: 3 × 7\n",
+       "\\begin{tabular}{r|lllllll}\n",
+       "  & Term & Df & Df.res & Sum Sq & Sum Sq.res & F value & Pr(>F)\\\\\n",
+       "  & <chr> & <dbl> & <dbl> & <dbl> & <dbl> & <dbl> & <dbl>\\\\\n",
+       "\\hline\n",
+       "\tretriever & retriever        & 1 & 232 &  41088.97 & 1037631.8 &   9.18692 & 2.714084e-03\\\\\n",
+       "\treader & reader           & 1 & 232 & 790427.81 &  301414.1 & 608.39633 & 8.802730e-67\\\\\n",
+       "\tretriever:reader & retriever:reader & 1 & 232 & 101903.46 &  983331.4 &  24.04235 & 1.771995e-06\\\\\n",
+       "\\end{tabular}\n"
+      ],
+      "text/markdown": [
+       "\n",
+       "A anova.art: 3 × 7\n",
+       "\n",
+       "| <!--/--> | Term &lt;chr&gt; | Df &lt;dbl&gt; | Df.res &lt;dbl&gt; | Sum Sq &lt;dbl&gt; | Sum Sq.res &lt;dbl&gt; | F value &lt;dbl&gt; | Pr(&gt;F) &lt;dbl&gt; |\n",
+       "|---|---|---|---|---|---|---|---|\n",
+       "| retriever | retriever        | 1 | 232 |  41088.97 | 1037631.8 |   9.18692 | 2.714084e-03 |\n",
+       "| reader | reader           | 1 | 232 | 790427.81 |  301414.1 | 608.39633 | 8.802730e-67 |\n",
+       "| retriever:reader | retriever:reader | 1 | 232 | 101903.46 |  983331.4 |  24.04235 | 1.771995e-06 |\n",
+       "\n"
+      ],
+      "text/plain": [
+       "                 Term             Df Df.res Sum Sq    Sum Sq.res F value  \n",
+       "retriever        retriever        1  232     41088.97 1037631.8    9.18692\n",
+       "reader           reader           1  232    790427.81  301414.1  608.39633\n",
+       "retriever:reader retriever:reader 1  232    101903.46  983331.4   24.04235\n",
+       "                 Pr(>F)      \n",
+       "retriever        2.714084e-03\n",
+       "reader           8.802730e-67\n",
+       "retriever:reader 1.771995e-06"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "NOTE: Results may be misleading due to involvement in interactions\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       " contrast   estimate   SE  df t.ratio p.value\n",
+       " es - faiss     26.4 8.71 232   3.031  0.0027\n",
+       "\n",
+       "Results are averaged over the levels of: reader "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "NOTE: Results may be misleading due to involvement in interactions\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       " contrast         estimate   SE  df t.ratio p.value\n",
+       " dpr - longformer     -116 4.69 232 -24.666  <.0001\n",
+       "\n",
+       "Results are averaged over the levels of: retriever "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "model.acc <- art(time ~ retriever * reader, data = timings_read)\n",
+    "anova(model.acc)\n",
+    "art.con(model.acc, ~ retriever)\n",
+    "art.con(model.acc, ~ reader)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "vscode": {
+     "languageId": "r"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table class=\"dataframe\">\n",
+       "<caption>A anova.art: 3 × 7</caption>\n",
+       "<thead>\n",
+       "\t<tr><th></th><th scope=col>Term</th><th scope=col>Df</th><th scope=col>Df.res</th><th scope=col>Sum Sq</th><th scope=col>Sum Sq.res</th><th scope=col>F value</th><th scope=col>Pr(&gt;F)</th></tr>\n",
+       "\t<tr><th></th><th scope=col>&lt;chr&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "\t<tr><th scope=row>retriever</th><td>retriever       </td><td>1</td><td>232</td><td>821516</td><td>240071.9</td><td>793.8944</td><td>7.630526e-77</td></tr>\n",
+       "\t<tr><th scope=row>reader</th><td>reader          </td><td>1</td><td>232</td><td>821516</td><td>214935.3</td><td>886.7398</td><td>3.256422e-81</td></tr>\n",
+       "\t<tr><th scope=row>retriever:reader</th><td>retriever:reader</td><td>1</td><td>232</td><td>821516</td><td>215501.6</td><td>884.4096</td><td>4.148583e-81</td></tr>\n",
+       "</tbody>\n",
+       "</table>\n"
+      ],
+      "text/latex": [
+       "A anova.art: 3 × 7\n",
+       "\\begin{tabular}{r|lllllll}\n",
+       "  & Term & Df & Df.res & Sum Sq & Sum Sq.res & F value & Pr(>F)\\\\\n",
+       "  & <chr> & <dbl> & <dbl> & <dbl> & <dbl> & <dbl> & <dbl>\\\\\n",
+       "\\hline\n",
+       "\tretriever & retriever        & 1 & 232 & 821516 & 240071.9 & 793.8944 & 7.630526e-77\\\\\n",
+       "\treader & reader           & 1 & 232 & 821516 & 214935.3 & 886.7398 & 3.256422e-81\\\\\n",
+       "\tretriever:reader & retriever:reader & 1 & 232 & 821516 & 215501.6 & 884.4096 & 4.148583e-81\\\\\n",
+       "\\end{tabular}\n"
+      ],
+      "text/markdown": [
+       "\n",
+       "A anova.art: 3 × 7\n",
+       "\n",
+       "| <!--/--> | Term &lt;chr&gt; | Df &lt;dbl&gt; | Df.res &lt;dbl&gt; | Sum Sq &lt;dbl&gt; | Sum Sq.res &lt;dbl&gt; | F value &lt;dbl&gt; | Pr(&gt;F) &lt;dbl&gt; |\n",
+       "|---|---|---|---|---|---|---|---|\n",
+       "| retriever | retriever        | 1 | 232 | 821516 | 240071.9 | 793.8944 | 7.630526e-77 |\n",
+       "| reader | reader           | 1 | 232 | 821516 | 214935.3 | 886.7398 | 3.256422e-81 |\n",
+       "| retriever:reader | retriever:reader | 1 | 232 | 821516 | 215501.6 | 884.4096 | 4.148583e-81 |\n",
+       "\n"
+      ],
+      "text/plain": [
+       "                 Term             Df Df.res Sum Sq Sum Sq.res F value \n",
+       "retriever        retriever        1  232    821516 240071.9   793.8944\n",
+       "reader           reader           1  232    821516 214935.3   886.7398\n",
+       "retriever:reader retriever:reader 1  232    821516 215501.6   884.4096\n",
+       "                 Pr(>F)      \n",
+       "retriever        7.630526e-77\n",
+       "reader           3.256422e-81\n",
+       "retriever:reader 4.148583e-81"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "NOTE: Results may be misleading due to involvement in interactions\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       " contrast   estimate   SE  df t.ratio p.value\n",
+       " es - faiss     -118 4.19 232 -28.176  <.0001\n",
+       "\n",
+       "Results are averaged over the levels of: reader "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "NOTE: Results may be misleading due to involvement in interactions\n",
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       " contrast         estimate   SE  df t.ratio p.value\n",
+       " dpr - longformer     -118 3.96 232 -29.778  <.0001\n",
+       "\n",
+       "Results are averaged over the levels of: retriever "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "model.acc <- art(time ~ retriever * reader, data = timings_retrieve)\n",
+    "anova(model.acc)\n",
+    "art.con(model.acc, ~ retriever)\n",
+    "art.con(model.acc, ~ reader)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "R",
+   "language": "R",
+   "name": "ir"
+  },
+  "language_info": {
+   "codemirror_mode": "r",
+   "file_extension": ".r",
+   "mimetype": "text/x-r-source",
+   "name": "R",
+   "pygments_lexer": "r",
+   "version": "4.1.2"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}