{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"# importing libraries\n",
"\n",
"import pandas as pd\n",
"from datasets import load_dataset\n",
"\n",
"import torch\n",
"from transformers import AutoTokenizer, Trainer, TrainingArguments, DefaultDataCollator, AutoModelForQuestionAnswering\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"# download and load the dataset splitting intor train and test set. Here only number of dataset is limited to 5000 \n",
"squad = load_dataset(\"squad\")"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"DatasetDict({\n",
" train: Dataset({\n",
" features: ['id', 'title', 'context', 'question', 'answers'],\n",
" num_rows: 87599\n",
" })\n",
" validation: Dataset({\n",
" features: ['id', 'title', 'context', 'question', 'answers'],\n",
" num_rows: 10570\n",
" })\n",
"})"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"squad"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"DatasetDict({\n",
" train: Dataset({\n",
" features: ['id', 'title', 'context', 'question', 'answers'],\n",
" num_rows: 87599\n",
" })\n",
" validation: Dataset({\n",
" features: ['id', 'title', 'context', 'question', 'answers'],\n",
" num_rows: 10570\n",
" })\n",
"})"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"squad"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'id': '5733be284776f41900661182', 'title': 'University_of_Notre_Dame', 'context': 'Architecturally, the school has a Catholic character. Atop the Main Building\\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.', 'question': 'To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France?', 'answers': {'text': ['Saint Bernadette Soubirous'], 'answer_start': [515]}}\n",
"{'id': '5733be284776f4190066117f', 'title': 'University_of_Notre_Dame', 'context': 'Architecturally, the school has a Catholic character. Atop the Main Building\\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.', 'question': 'What is in front of the Notre Dame Main Building?', 'answers': {'text': ['a copper statue of Christ'], 'answer_start': [188]}}\n",
"{'id': '5733be284776f41900661180', 'title': 'University_of_Notre_Dame', 'context': 'Architecturally, the school has a Catholic character. Atop the Main Building\\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.', 'question': 'The Basilica of the Sacred heart at Notre Dame is beside to which structure?', 'answers': {'text': ['the Main Building'], 'answer_start': [279]}}\n",
"{'id': '5733be284776f41900661181', 'title': 'University_of_Notre_Dame', 'context': 'Architecturally, the school has a Catholic character. Atop the Main Building\\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.', 'question': 'What is the Grotto at Notre Dame?', 'answers': {'text': ['a Marian place of prayer and reflection'], 'answer_start': [381]}}\n",
"{'id': '5733be284776f4190066117e', 'title': 'University_of_Notre_Dame', 'context': 'Architecturally, the school has a Catholic character. Atop the Main Building\\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend \"Venite Ad Me Omnes\". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.', 'question': 'What sits on top of the Main Building at Notre Dame?', 'answers': {'text': ['a golden statue of the Virgin Mary'], 'answer_start': [92]}}\n"
]
}
],
"source": [
"# Print the top 5 rows\n",
"for i in range(5):\n",
" print(squad[\"train\"][i])"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### several important fields \n",
"answers: the starting location of the answer token and the answer text. \n",
"context: background information from which the model needs to extract the answer. \n",
"question: the question a model should answer. "
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"tokenizer = AutoTokenizer.from_pretrained(\"distilbert-base-uncased\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Preprocessing Steps for Question Answering Tasks\n",
"\n",
"### Dealing with Long Contexts\n",
"\n",
"Some examples in a dataset may have a very long context that exceeds the maximum input length of the model. To address this:\n",
"\n",
"- Truncate only the context by setting `truncation=\"only_second\"`.\n",
"\n",
"### Mapping Answer Positions to Original Context\n",
"\n",
"To map the start and end positions of the answer to the original context:\n",
"\n",
"- Set `return_offset_mapping=True` during tokenization.\n",
"\n",
"### Finding Start and End Tokens of the Answer\n",
"\n",
"With the offset mapping, use the following steps to find the start and end tokens of the answer:\n",
"\n",
"- Utilize the `sequence_ids` method to determine which part of the offset corresponds to the question and which corresponds to the context."
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"def preprocess_function(examples):\n",
" questions = [q.strip() for q in examples[\"question\"]]\n",
" inputs = tokenizer(\n",
" questions,\n",
" examples[\"context\"],\n",
" max_length=384,\n",
" truncation=\"only_second\",\n",
" return_offsets_mapping=True,\n",
" padding=\"max_length\",\n",
" )\n",
"\n",
" offset_mapping = inputs.pop(\"offset_mapping\")\n",
" answers = examples[\"answers\"]\n",
" start_positions = []\n",
" end_positions = []\n",
"\n",
" for i, offset in enumerate(offset_mapping):\n",
" answer = answers[i]\n",
" start_char = answer[\"answer_start\"][0]\n",
" end_char = answer[\"answer_start\"][0] + len(answer[\"text\"][0])\n",
" sequence_ids = inputs.sequence_ids(i)\n",
"\n",
" # Find the start and end of the context\n",
" idx = 0\n",
" while sequence_ids[idx] != 1:\n",
" idx += 1\n",
" context_start = idx\n",
" while sequence_ids[idx] == 1:\n",
" idx += 1\n",
" context_end = idx - 1\n",
"\n",
" # If the answer is not fully inside the context, label it (0, 0)\n",
" if offset[context_start][0] > end_char or offset[context_end][1] < start_char:\n",
" start_positions.append(0)\n",
" end_positions.append(0)\n",
" else:\n",
" # Otherwise it's the start and end token positions\n",
" idx = context_start\n",
" while idx <= context_end and offset[idx][0] <= start_char:\n",
" idx += 1\n",
" start_positions.append(idx - 1)\n",
"\n",
" idx = context_end\n",
" while idx >= context_start and offset[idx][1] >= end_char:\n",
" idx -= 1\n",
" end_positions.append(idx + 1)\n",
"\n",
" inputs[\"start_positions\"] = start_positions\n",
" inputs[\"end_positions\"] = end_positions\n",
" return inputs"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Map: 0%| | 0/87599 [00:00