lingo judge init.

app.py

@@ -0,0 +1,6 @@
+import evaluate
+from evaluate.utils import launch_gradio_widget
+
+
+module = evaluate.load("maysonma/lingo_judge_metric")
+launch_gradio_widget(module)

constants.py

@@ -0,0 +1,23 @@
+# Source: https://github.com/wayveai/LingoQA/blob/main/benchmark/constants.py
+
+"""
+LingoQA datasets are stored in Google Cloud.
+This file provides the download link for the datasets, as well as reference keys for the data.
+"""
+from enum import Enum
+
+LINGOQA_TEST = "https://drive.usercontent.google.com/u/1/uc?id=1I8u6uYysQUstoVYZapyRQkXmOwr-AG3d&export=download"
+
+LINGO_JUDGE = "wayveai/Lingo-Judge"
+
+class Keys(str, Enum):
+    question_id = "question_id"
+    segment_id = "segment_id"
+    question = "question"
+    answer = "answer"
+    references = "references"
+    prediction = "prediction"
+    max_score = "max_score"
+    score = "score"
+    probability = "probability"
+    correct = "correct"

judge.py

@@ -0,0 +1,77 @@
+# Source: https://github.com/wayveai/LingoQA/blob/main/benchmark/judge.py
+
+import torch
+from torch import nn
+
+from typing import List
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+from constants import LINGO_JUDGE
+
+
+class LingoJudge(nn.Module):
+    """
+    LingoJudge is a textual classifier that evaluates the truthfulness of an answer on the LingoQA benchmark.
+    """
+
+    def __init__(self, pretrained_model=LINGO_JUDGE):
+        super().__init__()
+        self.tokenizer = AutoTokenizer.from_pretrained(pretrained_model, use_fast=True)
+        self.model = AutoModelForSequenceClassification.from_pretrained(pretrained_model).eval()
+
+    @torch.inference_mode()
+    def forward(self, question: str, references: List[str], prediction: str):
+        """
+        Inference function for textual classifier with multiple reference answers.
+        Args:
+            question: Input question.
+            references: List of references.
+            prediction: Model prediction.
+        Output:
+            scores: Score indicating truthfulness.
+        """
+        device = next(self.parameters()).device
+        texts = [
+            f"{self.tokenizer.cls_token}\nQuestion: {question}\nAnswer: {a_gt}\nStudent: {prediction}"
+            for a_gt in references
+        ]
+
+        encoded_input = self.tokenizer(
+            texts, return_tensors="pt", padding=True, truncation=True, max_length=128
+        )
+        encoded_input = {k: v.to(device) for k, v in encoded_input.items()}
+        output = self.model(**encoded_input)
+        scores = output.logits.squeeze(-1)
+        return scores
+
+    def compute(self, questions: List[str], references: List[List[str]], predictions: List[str]):
+        """
+        Compute maximum classifier metric. For multiple reference answers, selects the highest one.
+        Args:
+            questions: List of input questions.
+            references: List of lists, with multiple references per question supported.
+            predictions: List of model predictions.
+        Output:
+            scores: Score indicating truthfulness.
+        """
+        max_scores = []
+
+        for index, question in enumerate(questions):
+            references_preprocessed = [
+                self.preprocess(reference) for reference in references[index]
+            ]
+            prediction_preprocessed = self.preprocess(predictions[index])
+            scores = self.forward(question, references_preprocessed, prediction_preprocessed)
+            max_score = [max(scores)]
+            max_scores.extend(max_score)
+        return torch.Tensor(max_scores)
+
+    def preprocess(self, string: str):
+        """
+        Preprocessing function for consistency.
+        Args:
+            string: input string to be processed.
+        Output:
+            output: processed string with lower cases and trailing lines removed.
+        """
+        output = str(string).lower().lstrip().rstrip()
+        return output

lingo_judge_metric.py

@@ -0,0 +1,58 @@
+# Inspired by: https://huggingface.co/spaces/evaluate-metric/bleurt/blob/main/bleurt.py
+
+import datasets
+import evaluate
+import torch
+
+from judge import LingoJudge
+
+_CITATION = """
+@article{marcu2023lingoqa,
+  title={LingoQA: Video Question Answering for Autonomous Driving}, 
+  author={Ana-Maria Marcu and Long Chen and Jan Hünermann and Alice Karnsund and Benoit Hanotte and Prajwal Chidananda and Saurabh Nair and Vijay Badrinarayanan and Alex Kendall and Jamie Shotton and Oleg Sinavski},
+  journal={arXiv preprint arXiv:2312.14115},
+  year={2023},
+}
+"""
+
+_DESCRIPTION = """
+Lingo-Judge is an evaluation metric that aligns closely with human judgement on the LingoQA evaluation suite.
+
+See the project's README at https://github.com/wayveai/LingoQA for more information.
+"""
+
+_KWARGS_DESCRIPTION = """
+Lingo-Judge Score.
+
+Args:
+    'questions' (list of str): Input questions.
+    `predictions` (list of str): Model predictions.
+    `references` (list of list of str): Multiple references per question.
+
+Returns:
+    `scores` (list of float): Score indicating truthfulness.
+
+"""
+
+
+class LingoJudgeMetric(evaluate.Metric):
+    def _info(self):
+        return evaluate.MetricInfo(
+            description=_DESCRIPTION,
+            citation=_CITATION,
+            inputs_description=_KWARGS_DESCRIPTION,
+            features=datasets.Features(
+                {
+                    "questions": datasets.Value("string"),
+                    "predictions": datasets.Value("string"),
+                    "references": datasets.Sequence(datasets.Value("string")),
+                }
+            ),
+        )
+
+    def _download_and_prepare(self, dl_manager):
+        self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        self.scorer = LingoJudge().eval().to(self.device)
+
+    def _compute(self, questions, predictions, references):
+        return self.scorer.compute(questions, references, predictions)