Merge pull request #5 from arielhsieh8/milestone-3

AI_Milestone_3.ipynb

@@ -0,0 +1,714 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "view-in-github",
+        "colab_type": "text"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/arielhsieh8/cs-uy-4613-project/blob/milestone-3/AI_Milestone_3.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "MCO9jo5gyX2c",
+        "outputId": "b3fc4262-aa28-4363-d56e-b85a8fb29d3c"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
+            "Requirement already satisfied: transformers in /usr/local/lib/python3.9/dist-packages (4.28.1)\n",
+            "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from transformers) (2.27.1)\n",
+            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (0.13.3)\n",
+            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (23.1)\n",
+            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (6.0)\n",
+            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (1.22.4)\n",
+            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.9/dist-packages (from transformers) (4.65.0)\n",
+            "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers) (3.11.0)\n",
+            "Requirement already satisfied: huggingface-hub<1.0,>=0.11.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (0.13.4)\n",
+            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (2022.10.31)\n",
+            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.5.0)\n",
+            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (3.4)\n",
+            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2.0.12)\n",
+            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (1.26.15)\n",
+            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2022.12.7)\n",
+            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
+            "Requirement already satisfied: pandas in /usr/local/lib/python3.9/dist-packages (1.5.3)\n",
+            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/dist-packages (from pandas) (2.8.2)\n",
+            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas) (2022.7.1)\n",
+            "Requirement already satisfied: numpy>=1.20.3 in /usr/local/lib/python3.9/dist-packages (from pandas) (1.22.4)\n",
+            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.9/dist-packages (from python-dateutil>=2.8.1->pandas) (1.16.0)\n"
+          ]
+        }
+      ],
+      "source": [
+        "!pip install transformers\n",
+        "!pip install pandas\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "GHSa0Qb1xTvJ"
+      },
+      "outputs": [],
+      "source": [
+        "from sklearn.model_selection import train_test_split\n",
+        "import torch \n",
+        "from torch.utils.data import Dataset \n",
+        "from transformers import DistilBertTokenizerFast, DistilBertForSequenceClassification\n",
+        "from transformers import Trainer, TrainingArguments \n",
+        "import pandas as pd\n",
+        "import numpy as np\n",
+        "from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Sl36PcY2rxGX"
+      },
+      "outputs": [],
+      "source": [
+        "model_name = \"distilbert-base-uncased\"\n",
+        "\n",
+        "train_data = pd.read_csv('train.csv')\n",
+        "\n",
+        "train_data.drop([\"id\"], inplace=True, axis=1)\n",
+        "train_data.dropna()\n",
+        "\n",
+        "train_texts = train_data['comment_text'].tolist()\n",
+        "train_labels = train_data[['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']].values.tolist()\n",
+        "\n",
+        "train_texts, val_texts, train_labels, val_labels = train_test_split(train_texts[:100000],train_labels[:100000],test_size=0.2,random_state=42)\n",
+        "\n",
+        "class textDataset(Dataset):\n",
+        "\n",
+        "    def __init__(self, encodings, labels):\n",
+        "        self.encodings = encodings\n",
+        "        self.labels = torch.tensor(labels).float()\n",
+        "\n",
+        "    def __getitem__(self,index):\n",
+        "        item = {key: torch.tensor(val[index]) for key, val in self.encodings.items()}\n",
+        "        item['labels'] = torch.tensor(self.labels[index])\n",
+        "        return item\n",
+        "\n",
+        "    def __len__(self): \n",
+        "        return len(self.labels)\n",
+        "\n",
+        "\n",
+        "tokenizer = DistilBertTokenizerFast.from_pretrained(model_name,num_labels=6,problem_type=\"multi_label_classification\")\n",
+        "\n",
+        "train_encodings = tokenizer(train_texts,truncation=True,padding=True)\n",
+        "val_encodings = tokenizer(val_texts,truncation=True,padding=True)\n",
+        "\n",
+        "train_dataset = textDataset(train_encodings,train_labels)\n",
+        "val_dataset = textDataset(val_encodings,val_labels)\n",
+        "\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "8uyVppYpxJ7r",
+        "outputId": "6c7feff9-2b63-47fc-8fc8-78999b8a2d74"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_projector.weight', 'vocab_layer_norm.bias', 'vocab_layer_norm.weight', 'vocab_transform.bias', 'vocab_transform.weight', 'vocab_projector.bias']\n",
+            "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+            "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
+            "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.weight', 'pre_classifier.weight', 'pre_classifier.bias', 'classifier.bias']\n",
+            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+          ]
+        }
+      ],
+      "source": [
+        "training_args = TrainingArguments(\n",
+        "    output_dir='./results',\n",
+        "    num_train_epochs=2,\n",
+        "    per_device_train_batch_size=16,\n",
+        "    per_device_eval_batch_size=16,\n",
+        "    warmup_steps=500,\n",
+        "    learning_rate=5e-5,\n",
+        "    weight_decay=0.01,\n",
+        "    logging_dir='./logs',\n",
+        "    logging_steps=100,\n",
+        ")\n",
+        "\n",
+        "model = DistilBertForSequenceClassification.from_pretrained(model_name, num_labels=6,problem_type=\"multi_label_classification\")\n",
+        "\n",
+        "trainer = Trainer(\n",
+        "    model=model,\n",
+        "    args=training_args,\n",
+        "    train_dataset=train_dataset,\n",
+        "    eval_dataset=val_dataset,\n",
+        ")\n",
+        "\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 1000
+        },
+        "id": "lGigZhWtV0ld",
+        "outputId": "b2081e70-ed7c-4007-e231-3c9d269f398b"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "/usr/local/lib/python3.9/dist-packages/transformers/optimization.py:391: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
+            "  warnings.warn(\n",
+            "<ipython-input-3-a55db56b85e8>:21: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
+            "  item['labels'] = torch.tensor(self.labels[index])\n"
+          ]
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "<IPython.core.display.HTML object>"
+            ],
+            "text/html": [
+              "\n",
+              "    <div>\n",
+              "      \n",
+              "      <progress value='10000' max='10000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+              "      [10000/10000 2:00:23, Epoch 2/2]\n",
+              "    </div>\n",
+              "    <table border=\"1\" class=\"dataframe\">\n",
+              "  <thead>\n",
+              " <tr style=\"text-align: left;\">\n",
+              "      <th>Step</th>\n",
+              "      <th>Training Loss</th>\n",
+              "    </tr>\n",
+              "  </thead>\n",
+              "  <tbody>\n",
+              "    <tr>\n",
+              "      <td>100</td>\n",
+              "      <td>0.522000</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>200</td>\n",
+              "      <td>0.169400</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>300</td>\n",
+              "      <td>0.088900</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>400</td>\n",
+              "      <td>0.058000</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>500</td>\n",
+              "      <td>0.068900</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>600</td>\n",
+              "      <td>0.051600</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>700</td>\n",
+              "      <td>0.057400</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>800</td>\n",
+              "      <td>0.049300</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>900</td>\n",
+              "      <td>0.048100</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>1000</td>\n",
+              "      <td>0.062500</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>1100</td>\n",
+              "      <td>0.051300</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>1200</td>\n",
+              "      <td>0.050700</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>1300</td>\n",
+              "      <td>0.049000</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>1400</td>\n",
+              "      <td>0.047100</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>1500</td>\n",
+              "      <td>0.041500</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>1600</td>\n",
+              "      <td>0.049000</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>1700</td>\n",
+              "      <td>0.052800</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>1800</td>\n",
+              "      <td>0.049300</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>1900</td>\n",
+              "      <td>0.043500</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>2000</td>\n",
+              "      <td>0.047700</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>2100</td>\n",
+              "      <td>0.046600</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>2200</td>\n",
+              "      <td>0.045900</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>2300</td>\n",
+              "      <td>0.045900</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>2400</td>\n",
+              "      <td>0.042200</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>2500</td>\n",
+              "      <td>0.043100</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>2600</td>\n",
+              "      <td>0.044200</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>2700</td>\n",
+              "      <td>0.043900</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>2800</td>\n",
+              "      <td>0.042400</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>2900</td>\n",
+              "      <td>0.051700</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>3000</td>\n",
+              "      <td>0.049700</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>3100</td>\n",
+              "      <td>0.045700</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>3200</td>\n",
+              "      <td>0.047400</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>3300</td>\n",
+              "      <td>0.042800</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>3400</td>\n",
+              "      <td>0.042400</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>3500</td>\n",
+              "      <td>0.045200</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>3600</td>\n",
+              "      <td>0.047600</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>3700</td>\n",
+              "      <td>0.044800</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>3800</td>\n",
+              "      <td>0.045100</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>3900</td>\n",
+              "      <td>0.041900</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>4000</td>\n",
+              "      <td>0.039300</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>4100</td>\n",
+              "      <td>0.039500</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>4200</td>\n",
+              "      <td>0.044500</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>4300</td>\n",
+              "      <td>0.042700</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>4400</td>\n",
+              "      <td>0.039600</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>4500</td>\n",
+              "      <td>0.040300</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>4600</td>\n",
+              "      <td>0.044700</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>4700</td>\n",
+              "      <td>0.040700</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>4800</td>\n",
+              "      <td>0.036900</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>4900</td>\n",
+              "      <td>0.046200</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>5000</td>\n",
+              "      <td>0.040300</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>5100</td>\n",
+              "      <td>0.031600</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>5200</td>\n",
+              "      <td>0.029200</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>5300</td>\n",
+              "      <td>0.031900</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>5400</td>\n",
+              "      <td>0.030200</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>5500</td>\n",
+              "      <td>0.035700</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>5600</td>\n",
+              "      <td>0.028500</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>5700</td>\n",
+              "      <td>0.034600</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>5800</td>\n",
+              "      <td>0.027400</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>5900</td>\n",
+              "      <td>0.034700</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>6000</td>\n",
+              "      <td>0.038600</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>6100</td>\n",
+              "      <td>0.028500</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>6200</td>\n",
+              "      <td>0.030100</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>6300</td>\n",
+              "      <td>0.028300</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>6400</td>\n",
+              "      <td>0.029900</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>6500</td>\n",
+              "      <td>0.035500</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>6600</td>\n",
+              "      <td>0.031800</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>6700</td>\n",
+              "      <td>0.029200</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>6800</td>\n",
+              "      <td>0.031500</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>6900</td>\n",
+              "      <td>0.029700</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>7000</td>\n",
+              "      <td>0.030000</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>7100</td>\n",
+              "      <td>0.038800</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>7200</td>\n",
+              "      <td>0.030200</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>7300</td>\n",
+              "      <td>0.024700</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>7400</td>\n",
+              "      <td>0.034300</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>7500</td>\n",
+              "      <td>0.030400</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>7600</td>\n",
+              "      <td>0.029200</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>7700</td>\n",
+              "      <td>0.035600</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>7800</td>\n",
+              "      <td>0.033100</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>7900</td>\n",
+              "      <td>0.028300</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>8000</td>\n",
+              "      <td>0.027900</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>8100</td>\n",
+              "      <td>0.031400</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>8200</td>\n",
+              "      <td>0.038500</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>8300</td>\n",
+              "      <td>0.034400</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>8400</td>\n",
+              "      <td>0.030400</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>8500</td>\n",
+              "      <td>0.033000</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>8600</td>\n",
+              "      <td>0.034100</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>8700</td>\n",
+              "      <td>0.027100</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>8800</td>\n",
+              "      <td>0.029500</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>8900</td>\n",
+              "      <td>0.025700</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>9000</td>\n",
+              "      <td>0.029900</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>9100</td>\n",
+              "      <td>0.024000</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>9200</td>\n",
+              "      <td>0.028500</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>9300</td>\n",
+              "      <td>0.031400</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>9400</td>\n",
+              "      <td>0.028300</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>9500</td>\n",
+              "      <td>0.030500</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>9600</td>\n",
+              "      <td>0.025900</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>9700</td>\n",
+              "      <td>0.033600</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>9800</td>\n",
+              "      <td>0.030300</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>9900</td>\n",
+              "      <td>0.028700</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <td>10000</td>\n",
+              "      <td>0.022900</td>\n",
+              "    </tr>\n",
+              "  </tbody>\n",
+              "</table><p>"
+            ]
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "stream",
+          "name": "stderr",
+          "text": [
+            "<ipython-input-3-a55db56b85e8>:21: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
+            "  item['labels'] = torch.tensor(self.labels[index])\n",
+            "<ipython-input-3-a55db56b85e8>:21: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
+            "  item['labels'] = torch.tensor(self.labels[index])\n",
+            "<ipython-input-3-a55db56b85e8>:21: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
+            "  item['labels'] = torch.tensor(self.labels[index])\n",
+            "<ipython-input-3-a55db56b85e8>:21: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
+            "  item['labels'] = torch.tensor(self.labels[index])\n",
+            "<ipython-input-3-a55db56b85e8>:21: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
+            "  item['labels'] = torch.tensor(self.labels[index])\n",
+            "<ipython-input-3-a55db56b85e8>:21: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
+            "  item['labels'] = torch.tensor(self.labels[index])\n",
+            "<ipython-input-3-a55db56b85e8>:21: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
+            "  item['labels'] = torch.tensor(self.labels[index])\n",
+            "<ipython-input-3-a55db56b85e8>:21: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
+            "  item['labels'] = torch.tensor(self.labels[index])\n",
+            "<ipython-input-3-a55db56b85e8>:21: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
+            "  item['labels'] = torch.tensor(self.labels[index])\n",
+            "<ipython-input-3-a55db56b85e8>:21: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
+            "  item['labels'] = torch.tensor(self.labels[index])\n",
+            "<ipython-input-3-a55db56b85e8>:21: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
+            "  item['labels'] = torch.tensor(self.labels[index])\n",
+            "<ipython-input-3-a55db56b85e8>:21: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
+            "  item['labels'] = torch.tensor(self.labels[index])\n",
+            "<ipython-input-3-a55db56b85e8>:21: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
+            "  item['labels'] = torch.tensor(self.labels[index])\n",
+            "<ipython-input-3-a55db56b85e8>:21: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
+            "  item['labels'] = torch.tensor(self.labels[index])\n",
+            "<ipython-input-3-a55db56b85e8>:21: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
+            "  item['labels'] = torch.tensor(self.labels[index])\n",
+            "<ipython-input-3-a55db56b85e8>:21: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
+            "  item['labels'] = torch.tensor(self.labels[index])\n",
+            "<ipython-input-3-a55db56b85e8>:21: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
+            "  item['labels'] = torch.tensor(self.labels[index])\n",
+            "<ipython-input-3-a55db56b85e8>:21: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
+            "  item['labels'] = torch.tensor(self.labels[index])\n",
+            "<ipython-input-3-a55db56b85e8>:21: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
+            "  item['labels'] = torch.tensor(self.labels[index])\n"
+          ]
+        },
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "TrainOutput(global_step=10000, training_loss=0.045082428359985355, metrics={'train_runtime': 7226.7408, 'train_samples_per_second': 22.14, 'train_steps_per_second': 1.384, 'total_flos': 2.119629570048e+16, 'train_loss': 0.045082428359985355, 'epoch': 2.0})"
+            ]
+          },
+          "metadata": {},
+          "execution_count": 5
+        }
+      ],
+      "source": [
+        "trainer.train()"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "lowGDIRRV2Kk"
+      },
+      "outputs": [],
+      "source": [
+        "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
+        "\n",
+        "save_directory = \"saved\"\n",
+        "tokenizer.save_pretrained(save_directory)\n",
+        "model.save_pretrained(save_directory)\n",
+        "\n",
+        "tokenizer = AutoTokenizer.from_pretrained(save_directory)\n",
+        "model = AutoModelForSequenceClassification.from_pretrained(save_directory)"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "mount_file_id": "1SI5wXUWiK-4VnrwWn6Pq2r2e3pzK15mn",
+      "authorship_tag": "ABX9TyOWwkZmPEdojeBmja70X/+z",
+      "include_colab_link": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "accelerator": "GPU",
+    "gpuClass": "standard"
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}