Bug fix: right padding, and wrong model for final answer

02-autotrain.ipynb

@@ -18,9 +18,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "52543575-f92e-4038-ad13-30967f47eb7a",
-   "metadata": {},
+   "metadata": {
+    "jupyter": {
+     "is_executing": true
+    }
+   },
    "outputs": [],
    "source": [
     "import os\n",
@@ -37,6 +41,32 @@
     "## Config"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "6992324b-173c-4335-b557-cf78fbb2dd93",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "24ea5bd118ed4632a6ad859c4c976e66",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from huggingface_hub import login, get_token\n",
+    "login()"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "97c25070-775a-4fb1-9694-4579250686a6",
@@ -58,7 +88,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "id": "dc2a8514-51c1-404b-8cfa-6637cc810668",
    "metadata": {},
    "outputs": [],
@@ -80,15 +110,15 @@
     "            },\n",
     "        },\n",
     "    \"params\": {\n",
-    "        \"block_size\": 1024,\n",
-    "        \"model_max_length\": 1024,\n",
+    "        \"block_size\": 512,\n",
+    "        \"model_max_length\": 1500,\n",
     "        \"epochs\": 2,\n",
     "        \"batch_size\": 1,\n",
     "        \"lr\": 3e-5,\n",
     "        \"peft\": True,\n",
     "        \"quantization\": \"int4\",\n",
     "        \"target_modules\": \"all-linear\",\n",
-    "        \"padding\": \"left\",\n",
+    "        \"padding\": \"right\",\n",
     "        \"optimizer\": \"adamw_torch\",\n",
     "        \"scheduler\": \"linear\",\n",
     "        \"gradient_accumulation\": 8,\n",
@@ -96,7 +126,7 @@
     "        },\n",
     "    \"hub\": {\n",
     "        \"username\": \"derek-thomas\",\n",
-    "        \"token\": os.getenv('HF_TOKEN'),\n",
+    "        \"token\": get_token(),\n",
     "        \"push_to_hub\": True,\n",
     "        },\n",
     "}"
@@ -113,7 +143,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "id": "957eb2b7-feec-422f-ba46-b293d9a77c1b",
    "metadata": {},
    "outputs": [],
@@ -133,7 +163,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "id": "b86702bf-f494-4951-863e-be5b8462fbd1",
    "metadata": {},
    "outputs": [],
@@ -152,10 +182,47 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "id": "025ccd2f-de54-4ac2-9f36-f606876dcd3c",
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Running autotrain with config: ./autotrain_configs/conversation_RFA_gpt3_5.yml\n",
+      "INFO     | 2024-12-12 20:45:45 | autotrain.cli.autotrain:main:60 - Using AutoTrain configuration: ./autotrain_configs/conversation_RFA_gpt3_5.yml\n",
+      "INFO     | 2024-12-12 20:45:45 | autotrain.parser:__post_init__:170 - Running task: lm_training\n",
+      "INFO     | 2024-12-12 20:45:45 | autotrain.parser:__post_init__:171 - Using backend: spaces-l4x1\n",
+      "INFO     | 2024-12-12 20:45:45 | autotrain.parser:run:234 - {'model': 'mistralai/Mistral-7B-Instruct-v0.3', 'project_name': 'mistral-v03-poe-RFA-gpt3-5', 'data_path': 'derek-thomas/labeled-multiple-choice-explained-mistral-tokenized', 'train_split': 'train', 'valid_split': None, 'add_eos_token': True, 'block_size': 512, 'model_max_length': 1500, 'padding': 'right', 'trainer': 'sft', 'use_flash_attention_2': False, 'log': 'tensorboard', 'disable_gradient_checkpointing': False, 'logging_steps': -1, 'eval_strategy': 'epoch', 'save_total_limit': 1, 'auto_find_batch_size': False, 'mixed_precision': 'bf16', 'lr': 3e-05, 'epochs': 2, 'batch_size': 1, 'warmup_ratio': 0.1, 'gradient_accumulation': 8, 'optimizer': 'adamw_torch', 'scheduler': 'linear', 'weight_decay': 0.0, 'max_grad_norm': 1.0, 'seed': 42, 'chat_template': 'none', 'quantization': 'int4', 'target_modules': 'all-linear', 'merge_adapter': False, 'peft': True, 'lora_r': 16, 'lora_alpha': 32, 'lora_dropout': 0.05, 'model_ref': None, 'dpo_beta': 0.1, 'max_prompt_length': 128, 'max_completion_length': None, 'prompt_text_column': None, 'text_column': 'conversation_RFA_gpt3_5', 'rejected_text_column': None, 'push_to_hub': True, 'username': 'derek-thomas', 'token': '*****', 'unsloth': False, 'distributed_backend': None}\n",
+      "INFO     | 2024-12-12 20:45:52 | autotrain.parser:run:239 - Job ID: derek-thomas/autotrain-mistral-v03-poe-RFA-gpt3-5\n",
+      "Running autotrain with config: ./autotrain_configs/conversation_RFA_mistral.yml\n",
+      "INFO     | 2024-12-12 20:45:56 | autotrain.cli.autotrain:main:60 - Using AutoTrain configuration: ./autotrain_configs/conversation_RFA_mistral.yml\n",
+      "INFO     | 2024-12-12 20:45:56 | autotrain.parser:__post_init__:170 - Running task: lm_training\n",
+      "INFO     | 2024-12-12 20:45:56 | autotrain.parser:__post_init__:171 - Using backend: spaces-l4x1\n",
+      "INFO     | 2024-12-12 20:45:56 | autotrain.parser:run:234 - {'model': 'mistralai/Mistral-7B-Instruct-v0.3', 'project_name': 'mistral-v03-poe-RFA-mistral', 'data_path': 'derek-thomas/labeled-multiple-choice-explained-mistral-tokenized', 'train_split': 'train', 'valid_split': None, 'add_eos_token': True, 'block_size': 512, 'model_max_length': 1500, 'padding': 'right', 'trainer': 'sft', 'use_flash_attention_2': False, 'log': 'tensorboard', 'disable_gradient_checkpointing': False, 'logging_steps': -1, 'eval_strategy': 'epoch', 'save_total_limit': 1, 'auto_find_batch_size': False, 'mixed_precision': 'bf16', 'lr': 3e-05, 'epochs': 2, 'batch_size': 1, 'warmup_ratio': 0.1, 'gradient_accumulation': 8, 'optimizer': 'adamw_torch', 'scheduler': 'linear', 'weight_decay': 0.0, 'max_grad_norm': 1.0, 'seed': 42, 'chat_template': 'none', 'quantization': 'int4', 'target_modules': 'all-linear', 'merge_adapter': False, 'peft': True, 'lora_r': 16, 'lora_alpha': 32, 'lora_dropout': 0.05, 'model_ref': None, 'dpo_beta': 0.1, 'max_prompt_length': 128, 'max_completion_length': None, 'prompt_text_column': None, 'text_column': 'conversation_RFA_mistral', 'rejected_text_column': None, 'push_to_hub': True, 'username': 'derek-thomas', 'token': '*****', 'unsloth': False, 'distributed_backend': None}\n",
+      "INFO     | 2024-12-12 20:46:01 | autotrain.parser:run:239 - Job ID: derek-thomas/autotrain-mistral-v03-poe-RFA-mistral\n",
+      "Running autotrain with config: ./autotrain_configs/conversation_FAR_gpt3_5.yml\n",
+      "INFO     | 2024-12-12 20:46:05 | autotrain.cli.autotrain:main:60 - Using AutoTrain configuration: ./autotrain_configs/conversation_FAR_gpt3_5.yml\n",
+      "INFO     | 2024-12-12 20:46:05 | autotrain.parser:__post_init__:170 - Running task: lm_training\n",
+      "INFO     | 2024-12-12 20:46:05 | autotrain.parser:__post_init__:171 - Using backend: spaces-l4x1\n",
+      "INFO     | 2024-12-12 20:46:05 | autotrain.parser:run:234 - {'model': 'mistralai/Mistral-7B-Instruct-v0.3', 'project_name': 'mistral-v03-poe-FAR-gpt3-5', 'data_path': 'derek-thomas/labeled-multiple-choice-explained-mistral-tokenized', 'train_split': 'train', 'valid_split': None, 'add_eos_token': True, 'block_size': 512, 'model_max_length': 1500, 'padding': 'right', 'trainer': 'sft', 'use_flash_attention_2': False, 'log': 'tensorboard', 'disable_gradient_checkpointing': False, 'logging_steps': -1, 'eval_strategy': 'epoch', 'save_total_limit': 1, 'auto_find_batch_size': False, 'mixed_precision': 'bf16', 'lr': 3e-05, 'epochs': 2, 'batch_size': 1, 'warmup_ratio': 0.1, 'gradient_accumulation': 8, 'optimizer': 'adamw_torch', 'scheduler': 'linear', 'weight_decay': 0.0, 'max_grad_norm': 1.0, 'seed': 42, 'chat_template': 'none', 'quantization': 'int4', 'target_modules': 'all-linear', 'merge_adapter': False, 'peft': True, 'lora_r': 16, 'lora_alpha': 32, 'lora_dropout': 0.05, 'model_ref': None, 'dpo_beta': 0.1, 'max_prompt_length': 128, 'max_completion_length': None, 'prompt_text_column': None, 'text_column': 'conversation_FAR_gpt3_5', 'rejected_text_column': None, 'push_to_hub': True, 'username': 'derek-thomas', 'token': '*****', 'unsloth': False, 'distributed_backend': None}\n",
+      "INFO     | 2024-12-12 20:46:12 | autotrain.parser:run:239 - Job ID: derek-thomas/autotrain-mistral-v03-poe-FAR-gpt3-5\n",
+      "Running autotrain with config: ./autotrain_configs/conversation_FAR_mistral.yml\n",
+      "INFO     | 2024-12-12 20:46:16 | autotrain.cli.autotrain:main:60 - Using AutoTrain configuration: ./autotrain_configs/conversation_FAR_mistral.yml\n",
+      "INFO     | 2024-12-12 20:46:16 | autotrain.parser:__post_init__:170 - Running task: lm_training\n",
+      "INFO     | 2024-12-12 20:46:16 | autotrain.parser:__post_init__:171 - Using backend: spaces-l4x1\n",
+      "INFO     | 2024-12-12 20:46:16 | autotrain.parser:run:234 - {'model': 'mistralai/Mistral-7B-Instruct-v0.3', 'project_name': 'mistral-v03-poe-FAR-mistral', 'data_path': 'derek-thomas/labeled-multiple-choice-explained-mistral-tokenized', 'train_split': 'train', 'valid_split': None, 'add_eos_token': True, 'block_size': 512, 'model_max_length': 1500, 'padding': 'right', 'trainer': 'sft', 'use_flash_attention_2': False, 'log': 'tensorboard', 'disable_gradient_checkpointing': False, 'logging_steps': -1, 'eval_strategy': 'epoch', 'save_total_limit': 1, 'auto_find_batch_size': False, 'mixed_precision': 'bf16', 'lr': 3e-05, 'epochs': 2, 'batch_size': 1, 'warmup_ratio': 0.1, 'gradient_accumulation': 8, 'optimizer': 'adamw_torch', 'scheduler': 'linear', 'weight_decay': 0.0, 'max_grad_norm': 1.0, 'seed': 42, 'chat_template': 'none', 'quantization': 'int4', 'target_modules': 'all-linear', 'merge_adapter': False, 'peft': True, 'lora_r': 16, 'lora_alpha': 32, 'lora_dropout': 0.05, 'model_ref': None, 'dpo_beta': 0.1, 'max_prompt_length': 128, 'max_completion_length': None, 'prompt_text_column': None, 'text_column': 'conversation_FAR_mistral', 'rejected_text_column': None, 'push_to_hub': True, 'username': 'derek-thomas', 'token': '*****', 'unsloth': False, 'distributed_backend': None}\n",
+      "INFO     | 2024-12-12 20:46:22 | autotrain.parser:run:239 - Job ID: derek-thomas/autotrain-mistral-v03-poe-FAR-mistral\n",
+      "Running autotrain with config: ./autotrain_configs/conversation_FA.yml\n",
+      "INFO     | 2024-12-12 20:46:25 | autotrain.cli.autotrain:main:60 - Using AutoTrain configuration: ./autotrain_configs/conversation_FA.yml\n",
+      "INFO     | 2024-12-12 20:46:25 | autotrain.parser:__post_init__:170 - Running task: lm_training\n",
+      "INFO     | 2024-12-12 20:46:25 | autotrain.parser:__post_init__:171 - Using backend: spaces-l4x1\n",
+      "INFO     | 2024-12-12 20:46:25 | autotrain.parser:run:234 - {'model': 'mistralai/Mistral-7B-Instruct-v0.3', 'project_name': 'mistral-v03-poe-FA', 'data_path': 'derek-thomas/labeled-multiple-choice-explained-mistral-tokenized', 'train_split': 'train', 'valid_split': None, 'add_eos_token': True, 'block_size': 512, 'model_max_length': 1500, 'padding': 'right', 'trainer': 'sft', 'use_flash_attention_2': False, 'log': 'tensorboard', 'disable_gradient_checkpointing': False, 'logging_steps': -1, 'eval_strategy': 'epoch', 'save_total_limit': 1, 'auto_find_batch_size': False, 'mixed_precision': 'bf16', 'lr': 3e-05, 'epochs': 2, 'batch_size': 1, 'warmup_ratio': 0.1, 'gradient_accumulation': 8, 'optimizer': 'adamw_torch', 'scheduler': 'linear', 'weight_decay': 0.0, 'max_grad_norm': 1.0, 'seed': 42, 'chat_template': 'none', 'quantization': 'int4', 'target_modules': 'all-linear', 'merge_adapter': False, 'peft': True, 'lora_r': 16, 'lora_alpha': 32, 'lora_dropout': 0.05, 'model_ref': None, 'dpo_beta': 0.1, 'max_prompt_length': 128, 'max_completion_length': None, 'prompt_text_column': None, 'text_column': 'conversation_FA', 'rejected_text_column': None, 'push_to_hub': True, 'username': 'derek-thomas', 'token': '*****', 'unsloth': False, 'distributed_backend': None}\n",
+      "INFO     | 2024-12-12 20:46:31 | autotrain.parser:run:239 - Job ID: derek-thomas/autotrain-mistral-v03-poe-FA\n"
+     ]
+    }
+   ],
    "source": [
     "# Generate configs and run commands\n",
     "for project_suffix, text_column in zip(project_suffixes, text_columns):\n",
@@ -173,6 +240,14 @@
     "    print(f\"Running autotrain with config: {config_path}\")\n",
     "    subprocess.run([\"autotrain\", \"--config\", config_path])"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "67675837-2a38-4427-9186-32a25a970ff3",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

04-poe-eval.ipynb

@@ -104,7 +104,7 @@
     {
      "data": {
       "text/plain": [
-       "'derek-thomas/mistral-v03-poe-RFA-mistral,derek-thomas/mistral-v03-poe-FAR-mistral,derek-thomas/mistral-v03-poe-RFA-gpt3-5,derek-thomas/mistral-v03-poe-FAR-gpt3-5,derek-thomas/mistral-v03-poe-FAR'"
+       "'derek-thomas/mistral-v03-poe-RFA-mistral,derek-thomas/mistral-v03-poe-FAR-mistral,derek-thomas/mistral-v03-poe-RFA-gpt3-5,derek-thomas/mistral-v03-poe-FAR-gpt3-5,derek-thomas/mistral-v03-poe-FA'"
       ]
      },
      "execution_count": 3,
@@ -137,7 +137,7 @@
     "    },\n",
     "    'FA': {\n",
     "        'pydantic': FAModel,\n",
-    "        \"lora\": \"derek-thomas/mistral-v03-poe-FAR\",\n",
+    "        \"lora\": \"derek-thomas/mistral-v03-poe-FA\",\n",
     "        \"column\": 'user_prompt_FA',\n",
     "    },\n",
     "    'base': {\n",
@@ -162,7 +162,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "f9a6617489af4d86be59e614e3c505c4",
+       "model_id": "50dbecc676db4dc78dd1974d2f1a87dc",
        "version_major": 2,
        "version_minor": 0
       },
@@ -239,10 +239,9 @@
     "\n",
     "def get_my_endpoint():\n",
     "    name = f\"prompt-order-experiment\"\n",
-    "    namespace='HF-test-lab'\n",
+    "    namespace='derek-thomas'\n",
     "    try:\n",
     "        endpoint = get_inference_endpoint(name, namespace=namespace)\n",
-    "        endpoint.wait()\n",
     "    except:\n",
     "        # Custom Docker image details\n",
     "        custom_image = {\n",
@@ -279,7 +278,6 @@
     "            custom_image=custom_image,\n",
     "            secrets=secrets\n",
     "        )\n",
-    "        # endpoint.wait()\n",
     "            \n",
     "    print(\"Your model is ready to use!\")\n",
     "    endpoint.wait()\n",
@@ -297,8 +295,8 @@
      "output_type": "stream",
      "text": [
       "Your model is ready to use!\n",
-      "CPU times: user 22.3 ms, sys: 7.64 ms, total: 30 ms\n",
-      "Wall time: 2.07 s\n"
+      "CPU times: user 21.1 ms, sys: 10 ms, total: 31.1 ms\n",
+      "Wall time: 1.72 s\n"
      ]
     }
    ],
@@ -359,7 +357,7 @@
     {
      "data": {
       "text/plain": [
-       "'{\"Reasoning\": \"Busses are primarily used for transporting humans, so the correct answer is (b) Transporting humans. The other options are either incorrect (a, c, d, e, f, g, h) or not specific enough to the function of a bus (a, c, e, f, g, h).\", \"Final Answer\": \"b\"}'"
+       "'{\"Reasoning\": \"Busses are primarily used for transporting people, so the correct answer is (b) Transporting humans. The other options are not related to the function of a bus.\", \"Final Answer\": \"b\"}'"
       ]
      },
      "execution_count": 10,
@@ -388,7 +386,7 @@
     {
      "data": {
       "text/plain": [
-       "'{\"Reasoning\": \"Busses are primarily used for transporting humans, especially in urban areas, schools, and tourist destinations. They provide a means of public transportation, making it easier for people to travel to various locations without the need for personal vehicles. Therefore, the correct answer is (b) transporting humans.\", \"Final Answer\": \"b\"}'"
+       "'{\"Reasoning\": \"Busses are primarily used for transporting humans, especially in urban areas where public transportation is necessary. They provide a means of transportation for a large number of people at once, reducing traffic congestion and carbon emissions. Therefore, the correct answer is (b) transporting humans.\", \"Final Answer\": \"b\"}'"
       ]
      },
      "execution_count": 11,
@@ -400,7 +398,7 @@
     "key = 'RFA-gpt3-5'\n",
     "response = endpoint.client.text_generation(\n",
     "    prompt=user_prompt_RFA,\n",
-    "    max_new_tokens=575,\n",
+    "    max_new_tokens=OUTPUT_TOKENS,\n",
     "    adapter_id=experiments[key]['lora'],\n",
     "    grammar={\"type\": \"json\", \"value\": experiments[key]['pydantic'].schema()},\n",
     ")\n",
@@ -450,7 +448,7 @@
     {
      "data": {
       "text/plain": [
-       "'{\"Final Answer\": \"b\", \"Reasoning\": \"Busses are primarily used for transporting humans, especially in urban areas, to facilitate public transportation. They provide a means of transportation for a large number of people at once, reducing the number of vehicles on the road and helping to alleviate traffic congestion. They also serve as a protective shelter for passengers, shielding them from the elements during travel.\"}'"
+       "'{\"Final Answer\": \"b\", \"Reasoning\": \"Busses are primarily used for transporting humans, especially in urban areas, to facilitate commuting and travel. They are not used for protective shelter, helping other species benefit, transporting airplanes, serving as a backbone, communication, safe operation, or safe driving.\"}'"
       ]
      },
      "execution_count": 13,
@@ -479,7 +477,7 @@
     {
      "data": {
       "text/plain": [
-       "'{\"Final Answer\": \"b\", \"Reasoning\": \"Busses are primarily used for transporting humans. While they can provide a protective shelter and ensure safe operation, they are not used for transporting airplanes, helping other species benefit, serving as a backbone, or being used for communication.\"}'"
+       "'{\"Final Answer\": \"b\", \"Reasoning\": \"Busses are primarily used for transporting humans, especially in urban areas, to facilitate their travel from one place to another. The other options are incorrect because busses do not provide protective shelter, do not help other species benefit, are not used to transport airplanes, do not serve as a backbone, are not used for communication, and are not related to safe operation or driving.\"}'"
       ]
      },
      "execution_count": 14,
@@ -541,7 +539,7 @@
     {
      "data": {
       "text/plain": [
-       "'{\"Final Answer\": \"b\"}'"
+       "\"{'Final Answer': 'b'}\""
       ]
      },
      "execution_count": 16,
@@ -656,7 +654,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "f29c84d7a4654e9aa6af47cfe1bbde39",
+       "model_id": "be0bd3e278ae4d90a161918772ee71e8",
        "version_major": 2,
        "version_minor": 0
       },
@@ -765,7 +763,7 @@
     {
      "data": {
       "text/plain": [
-       "InferenceEndpoint(name='prompt-order-experiment', namespace='HF-test-lab', repository='mistralai/Mistral-7B-Instruct-v0.3', status='paused', url=None)"
+       "InferenceEndpoint(name='prompt-order-experiment', namespace='derek-thomas', repository='mistralai/Mistral-7B-Instruct-v0.3', status='paused', url=None)"
       ]
      },
      "execution_count": 19,
@@ -859,7 +857,7 @@
        "      <td>{\"Final Answer\": \"b\", \"Reasoning\": \"Busses are...</td>\n",
        "      <td>{\"Reasoning\": \"Busses are primarily used for t...</td>\n",
        "      <td>{\"Final Answer\": \"b\", \"Reasoning\": \"Busses are...</td>\n",
-       "      <td>{\"Final Answer\": \"b\"}</td>\n",
+       "      <td>{ \"Final Answer\": \"b\" }</td>\n",
        "      <td>{\"Final Answer\": \"b\"}</td>\n",
        "    </tr>\n",
        "    <tr>\n",
@@ -880,9 +878,9 @@
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
        "      <td>{\"Reasoning\": \"Global warming is primarily cau...</td>\n",
-       "      <td>{\"Final Answer\": \"a\", \"Reasoning\": \"The nucleu...</td>\n",
+       "      <td>{\"Final Answer\": \"g\", \"Reasoning\": \"Riding a b...</td>\n",
        "      <td>{\"Reasoning\": \"The nucleus of a cell (option a...</td>\n",
-       "      <td>{\"Final Answer\": \"a\", \"Reasoning\": \"The nucleu...</td>\n",
+       "      <td>{ \"Final Answer\": \"a\" ,\"Reasoning\": \"The nucle...</td>\n",
        "      <td>{\"Final Answer\": \"a\"}</td>\n",
        "      <td>{\"Final Answer\": \"a\"}</td>\n",
        "    </tr>\n",
@@ -904,10 +902,10 @@
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
        "      <td>{\"Reasoning\": \"The question asks for an organi...</td>\n",
-       "      <td>{\"Final Answer\": \"c\", \"Reasoning\": \"The proces...</td>\n",
-       "      <td>{\"Reasoning\": \"The process of converting chemi...</td>\n",
-       "      <td>{\"Final Answer\": \"e\", \"Reasoning\": \"Bacteria u...</td>\n",
-       "      <td>{\"Final Answer\": \"c\"}</td>\n",
+       "      <td>{\"Final Answer\": \"b\", \"Reasoning\": \"Cameras ar...</td>\n",
+       "      <td>{\"Reasoning\": \"The correct answer is (c) Cells...</td>\n",
+       "      <td>{\"Final Answer\": \"c\", \"Reasoning\": \"Cells are ...</td>\n",
+       "      <td>{\"Final Answer\": \"b\"}</td>\n",
        "      <td>{\"Final Answer\": \"c\"}</td>\n",
        "    </tr>\n",
        "    <tr>\n",
@@ -927,11 +925,11 @@
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
-       "      <td>{\"Reasoning\": \"Bacteria are microorganisms tha...</td>\n",
+       "      <td>{\"Reasoning\": \"The question asks about what is...</td>\n",
        "      <td>{\"Final Answer\": \"d\", \"Reasoning\": \"Bacteria c...</td>\n",
-       "      <td>{ \"Reasoning\": \"Bacteria can cause harm to var...</td>\n",
+       "      <td>{\"Reasoning\": \"Bacteria are microorganisms tha...</td>\n",
        "      <td>{\"Final Answer\": \"d\", \"Reasoning\": \"Bacteria c...</td>\n",
-       "      <td>{\"Final Answer\": \"d\"}</td>\n",
+       "      <td>{\"Final Answer\": \"e\"}</td>\n",
        "      <td>{\"Final Answer\": \"d\"}</td>\n",
        "    </tr>\n",
        "    <tr>\n",
@@ -951,11 +949,11 @@
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
-       "      <td>{\"Reasoning\": \"The question asks about the liv...</td>\n",
+       "      <td>{\"Reasoning\": \"The question asks for the livin...</td>\n",
        "      <td>{\"Final Answer\": \"a\", \"Reasoning\": \"Plants and...</td>\n",
-       "      <td>{\"Reasoning\": \"The question asks about the rel...</td>\n",
-       "      <td>{\"Final Answer\": \"f\", \"Reasoning\": \"Plants and...</td>\n",
-       "      <td>{\"Final Answer\": \"g\"}</td>\n",
+       "      <td>{\"Reasoning\": \"The question asks about the liv...</td>\n",
+       "      <td>{ \"Final Answer\": \"a\" ,\"Reasoning\": \"Plants an...</td>\n",
+       "      <td>{\"Final Answer\": \"a\"}</td>\n",
        "      <td>{\"Final Answer\": \"g\"}</td>\n",
        "    </tr>\n",
        "    <tr>\n",
@@ -1002,7 +1000,7 @@
        "      <td>{\"Reasoning\": \"The question asks for a way to ...</td>\n",
        "      <td>{\"Final Answer\": \"g\", \"Reasoning\": \"Recycling ...</td>\n",
        "      <td>{\"Reasoning\": \"Mining, fossil fuels, deforesta...</td>\n",
-       "      <td>{\"Final Answer\": \"g\", \"Reasoning\": \"Recycling ...</td>\n",
+       "      <td>{ \"Final Answer\": \"g\" ,\"Reasoning\": \"Recycling...</td>\n",
        "      <td>{\"Final Answer\": \"g\"}</td>\n",
        "      <td>{\"Final Answer\": \"g\"}</td>\n",
        "    </tr>\n",
@@ -1025,9 +1023,9 @@
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
        "      <td>{\"Reasoning\": \"The question asks for a term th...</td>\n",
        "      <td>{\"Final Answer\": \"d\", \"Reasoning\": \"A drought ...</td>\n",
-       "      <td>{ \"Reasoning\": \"A drought is a prolonged perio...</td>\n",
-       "      <td>{\"Final Answer\": \"d\", \"Reasoning\": \"A drought ...</td>\n",
-       "      <td>{\"Final Answer\": \"d\"}</td>\n",
+       "      <td>{\"Reasoning\": \"A drought is a prolonged period...</td>\n",
+       "      <td>{ \"Final Answer\": \"d\" ,\"Reasoning\": \"A drought...</td>\n",
+       "      <td>{\"Final Answer\": \"a\"}</td>\n",
        "      <td>{\"Final Answer\": \"d\"}</td>\n",
        "    </tr>\n",
        "    <tr>\n",
@@ -1051,7 +1049,7 @@
        "      <td>{\"Final Answer\": \"e\", \"Reasoning\": \"Ingestion ...</td>\n",
        "      <td>{\"Reasoning\": \"Ingestion is the process of tak...</td>\n",
        "      <td>{\"Final Answer\": \"e\", \"Reasoning\": \"Ingestion ...</td>\n",
-       "      <td>{\"Final Answer\": \"d\"}</td>\n",
+       "      <td>{\"Final Answer\": \"e\"}</td>\n",
        "      <td>{\"Final Answer\": \"d\"}</td>\n",
        "    </tr>\n",
        "    <tr>\n",
@@ -1074,7 +1072,7 @@
        "      <td>{\"Reasoning\": \"Ultraviolet (UV) light is a typ...</td>\n",
        "      <td>{\"Final Answer\": \"b\", \"Reasoning\": \"Ultraviole...</td>\n",
        "      <td>{\"Reasoning\": \"Ultraviolet (UV) light is a typ...</td>\n",
-       "      <td>{\"Final Answer\": \"d\", \"Reasoning\": \"Ultraviole...</td>\n",
+       "      <td>{ \"Final Answer\": \"b\" ,\"Reasoning\": \"Ultraviol...</td>\n",
        "      <td>{\"Final Answer\": \"b\"}</td>\n",
        "      <td>{\"Final Answer\": \"b\"}</td>\n",
        "    </tr>\n",
@@ -1095,11 +1093,11 @@
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
-       "      <td>{\"Reasoning\": \"The question asks for something...</td>\n",
+       "      <td>{\"Reasoning\": \"The correct answer is 'Exercise...</td>\n",
        "      <td>{\"Final Answer\": \"c\", \"Reasoning\": \"Running is...</td>\n",
        "      <td>{\"Reasoning\": \"A body's strength is primarily ...</td>\n",
        "      <td>{\"Final Answer\": \"c\", \"Reasoning\": \"Running is...</td>\n",
-       "      <td>{\"Final Answer\": \"c\"}</td>\n",
+       "      <td>{\"Final Answer\": \"f\"}</td>\n",
        "      <td>{\"Final Answer\": \"c\"}</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
@@ -1268,19 +1266,19 @@
        "0     {\"Reasoning\": \"Busses are primarily used for t...   \n",
        "1     {\"Reasoning\": \"Global warming is primarily cau...   \n",
        "2     {\"Reasoning\": \"The question asks for an organi...   \n",
-       "3     {\"Reasoning\": \"Bacteria are microorganisms tha...   \n",
-       "4     {\"Reasoning\": \"The question asks about the liv...   \n",
+       "3     {\"Reasoning\": \"The question asks about what is...   \n",
+       "4     {\"Reasoning\": \"The question asks for the livin...   \n",
        "...                                                 ...   \n",
        "1678  {\"Reasoning\": \"The question asks for a way to ...   \n",
        "1679  {\"Reasoning\": \"The question asks for a term th...   \n",
        "1680  {\"Reasoning\": \"Ingestion is the process of tak...   \n",
        "1681  {\"Reasoning\": \"Ultraviolet (UV) light is a typ...   \n",
-       "1682  {\"Reasoning\": \"The question asks for something...   \n",
+       "1682  {\"Reasoning\": \"The correct answer is 'Exercise...   \n",
        "\n",
        "                                  responses_FAR_mistral  \\\n",
        "0     {\"Final Answer\": \"b\", \"Reasoning\": \"Busses are...   \n",
-       "1     {\"Final Answer\": \"a\", \"Reasoning\": \"The nucleu...   \n",
-       "2     {\"Final Answer\": \"c\", \"Reasoning\": \"The proces...   \n",
+       "1     {\"Final Answer\": \"g\", \"Reasoning\": \"Riding a b...   \n",
+       "2     {\"Final Answer\": \"b\", \"Reasoning\": \"Cameras ar...   \n",
        "3     {\"Final Answer\": \"d\", \"Reasoning\": \"Bacteria c...   \n",
        "4     {\"Final Answer\": \"a\", \"Reasoning\": \"Plants and...   \n",
        "...                                                 ...   \n",
@@ -1293,41 +1291,41 @@
        "                                   responses_RFA_gpt3_5  \\\n",
        "0     {\"Reasoning\": \"Busses are primarily used for t...   \n",
        "1     {\"Reasoning\": \"The nucleus of a cell (option a...   \n",
-       "2     {\"Reasoning\": \"The process of converting chemi...   \n",
-       "3     { \"Reasoning\": \"Bacteria can cause harm to var...   \n",
-       "4     {\"Reasoning\": \"The question asks about the rel...   \n",
+       "2     {\"Reasoning\": \"The correct answer is (c) Cells...   \n",
+       "3     {\"Reasoning\": \"Bacteria are microorganisms tha...   \n",
+       "4     {\"Reasoning\": \"The question asks about the liv...   \n",
        "...                                                 ...   \n",
        "1678  {\"Reasoning\": \"Mining, fossil fuels, deforesta...   \n",
-       "1679  { \"Reasoning\": \"A drought is a prolonged perio...   \n",
+       "1679  {\"Reasoning\": \"A drought is a prolonged period...   \n",
        "1680  {\"Reasoning\": \"Ingestion is the process of tak...   \n",
        "1681  {\"Reasoning\": \"Ultraviolet (UV) light is a typ...   \n",
        "1682  {\"Reasoning\": \"A body's strength is primarily ...   \n",
        "\n",
        "                                   responses_FAR_gpt3_5  \\\n",
        "0     {\"Final Answer\": \"b\", \"Reasoning\": \"Busses are...   \n",
-       "1     {\"Final Answer\": \"a\", \"Reasoning\": \"The nucleu...   \n",
-       "2     {\"Final Answer\": \"e\", \"Reasoning\": \"Bacteria u...   \n",
+       "1     { \"Final Answer\": \"a\" ,\"Reasoning\": \"The nucle...   \n",
+       "2     {\"Final Answer\": \"c\", \"Reasoning\": \"Cells are ...   \n",
        "3     {\"Final Answer\": \"d\", \"Reasoning\": \"Bacteria c...   \n",
-       "4     {\"Final Answer\": \"f\", \"Reasoning\": \"Plants and...   \n",
+       "4     { \"Final Answer\": \"a\" ,\"Reasoning\": \"Plants an...   \n",
        "...                                                 ...   \n",
-       "1678  {\"Final Answer\": \"g\", \"Reasoning\": \"Recycling ...   \n",
-       "1679  {\"Final Answer\": \"d\", \"Reasoning\": \"A drought ...   \n",
+       "1678  { \"Final Answer\": \"g\" ,\"Reasoning\": \"Recycling...   \n",
+       "1679  { \"Final Answer\": \"d\" ,\"Reasoning\": \"A drought...   \n",
        "1680  {\"Final Answer\": \"e\", \"Reasoning\": \"Ingestion ...   \n",
-       "1681  {\"Final Answer\": \"d\", \"Reasoning\": \"Ultraviole...   \n",
+       "1681  { \"Final Answer\": \"b\" ,\"Reasoning\": \"Ultraviol...   \n",
        "1682  {\"Final Answer\": \"c\", \"Reasoning\": \"Running is...   \n",
        "\n",
-       "               responses_FA         responses_base  \n",
-       "0     {\"Final Answer\": \"b\"}  {\"Final Answer\": \"b\"}  \n",
-       "1     {\"Final Answer\": \"a\"}  {\"Final Answer\": \"a\"}  \n",
-       "2     {\"Final Answer\": \"c\"}  {\"Final Answer\": \"c\"}  \n",
-       "3     {\"Final Answer\": \"d\"}  {\"Final Answer\": \"d\"}  \n",
-       "4     {\"Final Answer\": \"g\"}  {\"Final Answer\": \"g\"}  \n",
-       "...                     ...                    ...  \n",
-       "1678  {\"Final Answer\": \"g\"}  {\"Final Answer\": \"g\"}  \n",
-       "1679  {\"Final Answer\": \"d\"}  {\"Final Answer\": \"d\"}  \n",
-       "1680  {\"Final Answer\": \"d\"}  {\"Final Answer\": \"d\"}  \n",
-       "1681  {\"Final Answer\": \"b\"}  {\"Final Answer\": \"b\"}  \n",
-       "1682  {\"Final Answer\": \"c\"}  {\"Final Answer\": \"c\"}  \n",
+       "                 responses_FA         responses_base  \n",
+       "0     { \"Final Answer\": \"b\" }  {\"Final Answer\": \"b\"}  \n",
+       "1       {\"Final Answer\": \"a\"}  {\"Final Answer\": \"a\"}  \n",
+       "2       {\"Final Answer\": \"b\"}  {\"Final Answer\": \"c\"}  \n",
+       "3       {\"Final Answer\": \"e\"}  {\"Final Answer\": \"d\"}  \n",
+       "4       {\"Final Answer\": \"a\"}  {\"Final Answer\": \"g\"}  \n",
+       "...                       ...                    ...  \n",
+       "1678    {\"Final Answer\": \"g\"}  {\"Final Answer\": \"g\"}  \n",
+       "1679    {\"Final Answer\": \"a\"}  {\"Final Answer\": \"d\"}  \n",
+       "1680    {\"Final Answer\": \"e\"}  {\"Final Answer\": \"d\"}  \n",
+       "1681    {\"Final Answer\": \"b\"}  {\"Final Answer\": \"b\"}  \n",
+       "1682    {\"Final Answer\": \"f\"}  {\"Final Answer\": \"c\"}  \n",
        "\n",
        "[1683 rows x 21 columns]"
       ]
@@ -1343,13 +1341,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 26,
    "id": "8619f9f5-9fe4-433e-b524-51c2b12e8d12",
    "metadata": {},
    "outputs": [],
    "source": [
     "def extract_final_answer(response):\n",
-    "    return json.loads(response).get(\"Final Answer\")\n",
+    "    try:\n",
+    "        answer = json.loads(response).get(\"Final Answer\")\n",
+    "    except:\n",
+    "        answer = 'x'\n",
+    "    return answer\n",
     "\n",
     "# Create new columns for predictions\n",
     "df['predictions_base'] = df['responses_base'].apply(extract_final_answer)\n",
@@ -1362,7 +1364,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 28,
    "id": "938cf2a3-2fed-42a3-82ec-a56cb0ea9f37",
    "metadata": {},
    "outputs": [
@@ -1370,12 +1372,12 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Base: \t\t\t\t\t\t45.28%\n",
-      "Final Answer: \t\t\t\t\t45.4%\n",
-      "Reasoning and then the Final Answer (Mistral): \t53.89%\n",
-      "Final Answer and then the Reasoning (Mistral): \t60.72%\n",
-      "Reasoning and then the Final Answer (GPT-3.5): \t59.06%\n",
-      "Final Answer and then the Reasoning (GPT-3.5): \t60.31%\n"
+      "Base: \t\t\t\t\t\t45.22%\n",
+      "Final Answer: \t\t\t\t\t64.53%\n",
+      "Reasoning and then the Final Answer (Mistral): \t55.02%\n",
+      "Final Answer and then the Reasoning (Mistral): \t61.79%\n",
+      "Reasoning and then the Final Answer (GPT-3.5): \t57.28%\n",
+      "Final Answer and then the Reasoning (GPT-3.5): \t61.62%\n"
      ]
     }
    ],
@@ -1392,7 +1394,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 29,
    "id": "83aae472-513b-43c3-9ee8-64d4cda775e0",
    "metadata": {},
    "outputs": [
@@ -1434,10 +1436,10 @@
        "      <th>responses_base</th>\n",
        "      <th>predictions_base</th>\n",
        "      <th>predictions_FA</th>\n",
-       "      <th>predictions_RFA_mistral</th>\n",
        "      <th>predictions_FAR_mistral</th>\n",
        "      <th>predictions_RFA_gpt3_5</th>\n",
        "      <th>predictions_FAR_gpt3_5</th>\n",
+       "      <th>predictions_RFA_mistral</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
@@ -1456,7 +1458,7 @@
        "      <td>...</td>\n",
        "      <td>{\"Reasoning\": \"Busses are primarily used for t...</td>\n",
        "      <td>{\"Final Answer\": \"b\", \"Reasoning\": \"Busses are...</td>\n",
-       "      <td>{\"Final Answer\": \"b\"}</td>\n",
+       "      <td>{ \"Final Answer\": \"b\" }</td>\n",
        "      <td>{\"Final Answer\": \"b\"}</td>\n",
        "      <td>b</td>\n",
        "      <td>b</td>\n",
@@ -1479,7 +1481,7 @@
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
        "      <td>...</td>\n",
        "      <td>{\"Reasoning\": \"The nucleus of a cell (option a...</td>\n",
-       "      <td>{\"Final Answer\": \"a\", \"Reasoning\": \"The nucleu...</td>\n",
+       "      <td>{ \"Final Answer\": \"a\" ,\"Reasoning\": \"The nucle...</td>\n",
        "      <td>{\"Final Answer\": \"a\"}</td>\n",
        "      <td>{\"Final Answer\": \"a\"}</td>\n",
        "      <td>a</td>\n",
@@ -1487,7 +1489,7 @@
        "      <td>g</td>\n",
        "      <td>a</td>\n",
        "      <td>a</td>\n",
-       "      <td>a</td>\n",
+       "      <td>g</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
@@ -1502,13 +1504,13 @@
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
        "      <td>...</td>\n",
-       "      <td>{\"Reasoning\": \"The process of converting chemi...</td>\n",
-       "      <td>{\"Final Answer\": \"e\", \"Reasoning\": \"Bacteria u...</td>\n",
-       "      <td>{\"Final Answer\": \"c\"}</td>\n",
+       "      <td>{\"Reasoning\": \"The correct answer is (c) Cells...</td>\n",
+       "      <td>{\"Final Answer\": \"c\", \"Reasoning\": \"Cells are ...</td>\n",
+       "      <td>{\"Final Answer\": \"b\"}</td>\n",
        "      <td>{\"Final Answer\": \"c\"}</td>\n",
        "      <td>c</td>\n",
-       "      <td>c</td>\n",
-       "      <td>e</td>\n",
+       "      <td>b</td>\n",
+       "      <td>b</td>\n",
        "      <td>c</td>\n",
        "      <td>c</td>\n",
        "      <td>e</td>\n",
@@ -1526,16 +1528,16 @@
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
        "      <td>...</td>\n",
-       "      <td>{ \"Reasoning\": \"Bacteria can cause harm to var...</td>\n",
+       "      <td>{\"Reasoning\": \"Bacteria are microorganisms tha...</td>\n",
        "      <td>{\"Final Answer\": \"d\", \"Reasoning\": \"Bacteria c...</td>\n",
+       "      <td>{\"Final Answer\": \"e\"}</td>\n",
        "      <td>{\"Final Answer\": \"d\"}</td>\n",
-       "      <td>{\"Final Answer\": \"d\"}</td>\n",
-       "      <td>d</td>\n",
-       "      <td>d</td>\n",
        "      <td>d</td>\n",
+       "      <td>e</td>\n",
        "      <td>d</td>\n",
        "      <td>d</td>\n",
        "      <td>d</td>\n",
+       "      <td>e</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>4</th>\n",
@@ -1550,16 +1552,16 @@
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
        "      <td>...</td>\n",
-       "      <td>{\"Reasoning\": \"The question asks about the rel...</td>\n",
-       "      <td>{\"Final Answer\": \"f\", \"Reasoning\": \"Plants and...</td>\n",
-       "      <td>{\"Final Answer\": \"g\"}</td>\n",
+       "      <td>{\"Reasoning\": \"The question asks about the liv...</td>\n",
+       "      <td>{ \"Final Answer\": \"a\" ,\"Reasoning\": \"Plants an...</td>\n",
+       "      <td>{\"Final Answer\": \"a\"}</td>\n",
        "      <td>{\"Final Answer\": \"g\"}</td>\n",
        "      <td>g</td>\n",
-       "      <td>g</td>\n",
-       "      <td>b</td>\n",
+       "      <td>a</td>\n",
        "      <td>a</td>\n",
        "      <td>f</td>\n",
-       "      <td>f</td>\n",
+       "      <td>a</td>\n",
+       "      <td>b</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>...</th>\n",
@@ -1599,7 +1601,7 @@
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
        "      <td>...</td>\n",
        "      <td>{\"Reasoning\": \"Mining, fossil fuels, deforesta...</td>\n",
-       "      <td>{\"Final Answer\": \"g\", \"Reasoning\": \"Recycling ...</td>\n",
+       "      <td>{ \"Final Answer\": \"g\" ,\"Reasoning\": \"Recycling...</td>\n",
        "      <td>{\"Final Answer\": \"g\"}</td>\n",
        "      <td>{\"Final Answer\": \"g\"}</td>\n",
        "      <td>g</td>\n",
@@ -1622,15 +1624,15 @@
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
        "      <td>...</td>\n",
-       "      <td>{ \"Reasoning\": \"A drought is a prolonged perio...</td>\n",
-       "      <td>{\"Final Answer\": \"d\", \"Reasoning\": \"A drought ...</td>\n",
-       "      <td>{\"Final Answer\": \"d\"}</td>\n",
+       "      <td>{\"Reasoning\": \"A drought is a prolonged period...</td>\n",
+       "      <td>{ \"Final Answer\": \"d\" ,\"Reasoning\": \"A drought...</td>\n",
+       "      <td>{\"Final Answer\": \"a\"}</td>\n",
        "      <td>{\"Final Answer\": \"d\"}</td>\n",
        "      <td>d</td>\n",
+       "      <td>a</td>\n",
        "      <td>d</td>\n",
        "      <td>d</td>\n",
        "      <td>d</td>\n",
-       "      <td>f</td>\n",
        "      <td>d</td>\n",
        "    </tr>\n",
        "    <tr>\n",
@@ -1648,14 +1650,14 @@
        "      <td>...</td>\n",
        "      <td>{\"Reasoning\": \"Ingestion is the process of tak...</td>\n",
        "      <td>{\"Final Answer\": \"e\", \"Reasoning\": \"Ingestion ...</td>\n",
+       "      <td>{\"Final Answer\": \"e\"}</td>\n",
        "      <td>{\"Final Answer\": \"d\"}</td>\n",
-       "      <td>{\"Final Answer\": \"d\"}</td>\n",
-       "      <td>d</td>\n",
        "      <td>d</td>\n",
        "      <td>e</td>\n",
        "      <td>e</td>\n",
        "      <td>e</td>\n",
        "      <td>e</td>\n",
+       "      <td>c</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1681</th>\n",
@@ -1671,7 +1673,7 @@
        "      <td>&lt;s&gt;[INST] Answer the Question and include your...</td>\n",
        "      <td>...</td>\n",
        "      <td>{\"Reasoning\": \"Ultraviolet (UV) light is a typ...</td>\n",
-       "      <td>{\"Final Answer\": \"d\", \"Reasoning\": \"Ultraviole...</td>\n",
+       "      <td>{ \"Final Answer\": \"b\" ,\"Reasoning\": \"Ultraviol...</td>\n",
        "      <td>{\"Final Answer\": \"b\"}</td>\n",
        "      <td>{\"Final Answer\": \"b\"}</td>\n",
        "      <td>b</td>\n",
@@ -1679,7 +1681,7 @@
        "      <td>b</td>\n",
        "      <td>b</td>\n",
        "      <td>b</td>\n",
-       "      <td>d</td>\n",
+       "      <td>b</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1682</th>\n",
@@ -1696,13 +1698,13 @@
        "      <td>...</td>\n",
        "      <td>{\"Reasoning\": \"A body's strength is primarily ...</td>\n",
        "      <td>{\"Final Answer\": \"c\", \"Reasoning\": \"Running is...</td>\n",
-       "      <td>{\"Final Answer\": \"c\"}</td>\n",
+       "      <td>{\"Final Answer\": \"f\"}</td>\n",
        "      <td>{\"Final Answer\": \"c\"}</td>\n",
        "      <td>c</td>\n",
-       "      <td>c</td>\n",
        "      <td>f</td>\n",
        "      <td>c</td>\n",
-       "      <td>f</td>\n",
+       "      <td>d</td>\n",
+       "      <td>c</td>\n",
        "      <td>c</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
@@ -1818,72 +1820,72 @@
        "                                   responses_RFA_gpt3_5  \\\n",
        "0     {\"Reasoning\": \"Busses are primarily used for t...   \n",
        "1     {\"Reasoning\": \"The nucleus of a cell (option a...   \n",
-       "2     {\"Reasoning\": \"The process of converting chemi...   \n",
-       "3     { \"Reasoning\": \"Bacteria can cause harm to var...   \n",
-       "4     {\"Reasoning\": \"The question asks about the rel...   \n",
+       "2     {\"Reasoning\": \"The correct answer is (c) Cells...   \n",
+       "3     {\"Reasoning\": \"Bacteria are microorganisms tha...   \n",
+       "4     {\"Reasoning\": \"The question asks about the liv...   \n",
        "...                                                 ...   \n",
        "1678  {\"Reasoning\": \"Mining, fossil fuels, deforesta...   \n",
-       "1679  { \"Reasoning\": \"A drought is a prolonged perio...   \n",
+       "1679  {\"Reasoning\": \"A drought is a prolonged period...   \n",
        "1680  {\"Reasoning\": \"Ingestion is the process of tak...   \n",
        "1681  {\"Reasoning\": \"Ultraviolet (UV) light is a typ...   \n",
        "1682  {\"Reasoning\": \"A body's strength is primarily ...   \n",
        "\n",
        "                                   responses_FAR_gpt3_5  \\\n",
        "0     {\"Final Answer\": \"b\", \"Reasoning\": \"Busses are...   \n",
-       "1     {\"Final Answer\": \"a\", \"Reasoning\": \"The nucleu...   \n",
-       "2     {\"Final Answer\": \"e\", \"Reasoning\": \"Bacteria u...   \n",
+       "1     { \"Final Answer\": \"a\" ,\"Reasoning\": \"The nucle...   \n",
+       "2     {\"Final Answer\": \"c\", \"Reasoning\": \"Cells are ...   \n",
        "3     {\"Final Answer\": \"d\", \"Reasoning\": \"Bacteria c...   \n",
-       "4     {\"Final Answer\": \"f\", \"Reasoning\": \"Plants and...   \n",
+       "4     { \"Final Answer\": \"a\" ,\"Reasoning\": \"Plants an...   \n",
        "...                                                 ...   \n",
-       "1678  {\"Final Answer\": \"g\", \"Reasoning\": \"Recycling ...   \n",
-       "1679  {\"Final Answer\": \"d\", \"Reasoning\": \"A drought ...   \n",
+       "1678  { \"Final Answer\": \"g\" ,\"Reasoning\": \"Recycling...   \n",
+       "1679  { \"Final Answer\": \"d\" ,\"Reasoning\": \"A drought...   \n",
        "1680  {\"Final Answer\": \"e\", \"Reasoning\": \"Ingestion ...   \n",
-       "1681  {\"Final Answer\": \"d\", \"Reasoning\": \"Ultraviole...   \n",
+       "1681  { \"Final Answer\": \"b\" ,\"Reasoning\": \"Ultraviol...   \n",
        "1682  {\"Final Answer\": \"c\", \"Reasoning\": \"Running is...   \n",
        "\n",
-       "               responses_FA         responses_base predictions_base  \\\n",
-       "0     {\"Final Answer\": \"b\"}  {\"Final Answer\": \"b\"}                b   \n",
-       "1     {\"Final Answer\": \"a\"}  {\"Final Answer\": \"a\"}                a   \n",
-       "2     {\"Final Answer\": \"c\"}  {\"Final Answer\": \"c\"}                c   \n",
-       "3     {\"Final Answer\": \"d\"}  {\"Final Answer\": \"d\"}                d   \n",
-       "4     {\"Final Answer\": \"g\"}  {\"Final Answer\": \"g\"}                g   \n",
-       "...                     ...                    ...              ...   \n",
-       "1678  {\"Final Answer\": \"g\"}  {\"Final Answer\": \"g\"}                g   \n",
-       "1679  {\"Final Answer\": \"d\"}  {\"Final Answer\": \"d\"}                d   \n",
-       "1680  {\"Final Answer\": \"d\"}  {\"Final Answer\": \"d\"}                d   \n",
-       "1681  {\"Final Answer\": \"b\"}  {\"Final Answer\": \"b\"}                b   \n",
-       "1682  {\"Final Answer\": \"c\"}  {\"Final Answer\": \"c\"}                c   \n",
+       "                 responses_FA         responses_base predictions_base  \\\n",
+       "0     { \"Final Answer\": \"b\" }  {\"Final Answer\": \"b\"}                b   \n",
+       "1       {\"Final Answer\": \"a\"}  {\"Final Answer\": \"a\"}                a   \n",
+       "2       {\"Final Answer\": \"b\"}  {\"Final Answer\": \"c\"}                c   \n",
+       "3       {\"Final Answer\": \"e\"}  {\"Final Answer\": \"d\"}                d   \n",
+       "4       {\"Final Answer\": \"a\"}  {\"Final Answer\": \"g\"}                g   \n",
+       "...                       ...                    ...              ...   \n",
+       "1678    {\"Final Answer\": \"g\"}  {\"Final Answer\": \"g\"}                g   \n",
+       "1679    {\"Final Answer\": \"a\"}  {\"Final Answer\": \"d\"}                d   \n",
+       "1680    {\"Final Answer\": \"e\"}  {\"Final Answer\": \"d\"}                d   \n",
+       "1681    {\"Final Answer\": \"b\"}  {\"Final Answer\": \"b\"}                b   \n",
+       "1682    {\"Final Answer\": \"f\"}  {\"Final Answer\": \"c\"}                c   \n",
        "\n",
-       "     predictions_FA predictions_RFA_mistral predictions_FAR_mistral  \\\n",
-       "0                 b                       b                       b   \n",
-       "1                 a                       g                       a   \n",
-       "2                 c                       e                       c   \n",
-       "3                 d                       d                       d   \n",
-       "4                 g                       b                       a   \n",
-       "...             ...                     ...                     ...   \n",
-       "1678              g                       g                       g   \n",
-       "1679              d                       d                       d   \n",
-       "1680              d                       e                       e   \n",
-       "1681              b                       b                       b   \n",
-       "1682              c                       f                       c   \n",
+       "     predictions_FA predictions_FAR_mistral predictions_RFA_gpt3_5  \\\n",
+       "0                 b                       b                      b   \n",
+       "1                 a                       g                      a   \n",
+       "2                 b                       b                      c   \n",
+       "3                 e                       d                      d   \n",
+       "4                 a                       a                      f   \n",
+       "...             ...                     ...                    ...   \n",
+       "1678              g                       g                      g   \n",
+       "1679              a                       d                      d   \n",
+       "1680              e                       e                      e   \n",
+       "1681              b                       b                      b   \n",
+       "1682              f                       c                      d   \n",
        "\n",
-       "     predictions_RFA_gpt3_5 predictions_FAR_gpt3_5  \n",
-       "0                         b                      b  \n",
-       "1                         a                      a  \n",
-       "2                         c                      e  \n",
-       "3                         d                      d  \n",
-       "4                         f                      f  \n",
-       "...                     ...                    ...  \n",
-       "1678                      g                      g  \n",
-       "1679                      f                      d  \n",
-       "1680                      e                      e  \n",
-       "1681                      b                      d  \n",
-       "1682                      f                      c  \n",
+       "     predictions_FAR_gpt3_5 predictions_RFA_mistral  \n",
+       "0                         b                       b  \n",
+       "1                         a                       g  \n",
+       "2                         c                       e  \n",
+       "3                         d                       e  \n",
+       "4                         a                       b  \n",
+       "...                     ...                     ...  \n",
+       "1678                      g                       g  \n",
+       "1679                      d                       d  \n",
+       "1680                      e                       c  \n",
+       "1681                      b                       b  \n",
+       "1682                      c                       c  \n",
        "\n",
        "[1683 rows x 27 columns]"
       ]
      },
-     "execution_count": 24,
+     "execution_count": 29,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1894,14 +1896,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 30,
    "id": "45c08dd4-0b98-4e0f-b487-549f60518a4e",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "30c9a9f7656f4950a3fde9deaa6bf0ac",
+       "model_id": "23d5dbd0a91d436fb9920dfe81e4803a",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1915,7 +1917,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "bfc21892a7eb45caa097435b6026087d",
+       "model_id": "0b25bcb277574e8792b14e838a32fe25",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1929,10 +1931,10 @@
     {
      "data": {
       "text/plain": [
-       "CommitInfo(commit_url='https://huggingface.co/datasets/derek-thomas/labeled-multiple-choice-explained-mistral-results/commit/b2c01c5867b23afe06c0806f2886152864574946', commit_message='Upload dataset', commit_description='', oid='b2c01c5867b23afe06c0806f2886152864574946', pr_url=None, repo_url=RepoUrl('https://huggingface.co/datasets/derek-thomas/labeled-multiple-choice-explained-mistral-results', endpoint='https://huggingface.co', repo_type='dataset', repo_id='derek-thomas/labeled-multiple-choice-explained-mistral-results'), pr_revision=None, pr_num=None)"
+       "CommitInfo(commit_url='https://huggingface.co/datasets/derek-thomas/labeled-multiple-choice-explained-mistral-results/commit/796d0867b715f2fad05d6e54ad1e0e0504ca670c', commit_message='Upload dataset', commit_description='', oid='796d0867b715f2fad05d6e54ad1e0e0504ca670c', pr_url=None, repo_url=RepoUrl('https://huggingface.co/datasets/derek-thomas/labeled-multiple-choice-explained-mistral-results', endpoint='https://huggingface.co', repo_type='dataset', repo_id='derek-thomas/labeled-multiple-choice-explained-mistral-results'), pr_revision=None, pr_num=None)"
       ]
      },
-     "execution_count": 26,
+     "execution_count": 30,
      "metadata": {},
      "output_type": "execute_result"
     }

requirements.txt

@@ -8,4 +8,9 @@ scikit-learn
 lighteval[tensorboardX,adapters]
 nest_asyncio
 plotly
-ipywidgets
+ipywidgets
+
+# Reflex
+reflex
+reflex-ag-grid
+