| Step | \n", "Training Loss | \n", "Validation Loss | \n", "
|---|---|---|
| 19 | \n", "1.326300 | \n", "No log | \n", "
| 38 | \n", "1.270200 | \n", "No log | \n", "
| 57 | \n", "1.290100 | \n", "No log | \n", "
| 76 | \n", "1.166500 | \n", "No log | \n", "
" ] }, "metadata": {} }, { "output_type": "execute_result", "data": { "text/plain": [ "TrainOutput(global_step=93, training_loss=1.264249195334732, metrics={'train_runtime': 2513.5877, 'train_samples_per_second': 0.597, 'train_steps_per_second': 0.037, 'total_flos': 1.2725736297725952e+16, 'train_loss': 1.264249195334732, 'epoch': 2.98})" ] }, "metadata": {}, "execution_count": 19 } ] }, { "cell_type": "code", "source": [ "peft_model_path=\"./llama-dialogue-summarizer\"\n", "\n", "trainer.model.save_pretrained(peft_model_path)\n", "tokenizer.save_pretrained(peft_model_path)" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "qfptHpZVgPEp", "outputId": "6063f158-b3a6-41b7-ceec-fccbfe5929a0" }, "execution_count": null, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "('./llama-dialogue-summarizer/tokenizer_config.json',\n", " './llama-dialogue-summarizer/special_tokens_map.json',\n", " './llama-dialogue-summarizer/tokenizer.json')" ] }, "metadata": {}, "execution_count": 20 } ] }, { "cell_type": "code", "source": [ "!pip install huggingface_hub" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "CvOZN6Ww42N6", "outputId": "41683536-1863-418f-abaa-39642713a5c6" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Requirement already satisfied: huggingface_hub in /usr/local/lib/python3.10/dist-packages (0.19.4)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (3.13.1)\n", "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (2023.6.0)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (2.31.0)\n", "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (4.66.1)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (6.0.1)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (4.5.0)\n", "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface_hub) (23.2)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (3.3.2)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (3.6)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (2.0.7)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface_hub) (2023.11.17)\n" ] } ] }, { "cell_type": "code", "source": [ "!huggingface-cli login --token 'add-your-token'" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "NBItkxKj5ZBC", "outputId": "f80f117d-64cd-4617-c8d6-031b361ad4bd" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Token will not been saved to git credential helper. Pass `add_to_git_credential=True` if you want to set the git credential as well.\n", "Token is valid (permission: write).\n", "Your token has been saved to /root/.cache/huggingface/token\n", "Login successful\n" ] } ] }, { "cell_type": "code", "source": [ "!zip -r llama-dialogue-summarizer.zip /content/llama-dialogue-summarizer" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "a0oxng7R4QWn", "outputId": "a7c6e972-bbc1-4dac-fe7e-54c7c74b24d4" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ " adding: content/llama-dialogue-summarizer/ (stored 0%)\n", " adding: content/llama-dialogue-summarizer/tokenizer.json (deflated 74%)\n", " adding: content/llama-dialogue-summarizer/special_tokens_map.json (deflated 72%)\n", " adding: content/llama-dialogue-summarizer/adapter_model.safetensors (deflated 8%)\n", " adding: content/llama-dialogue-summarizer/README.md (deflated 66%)\n", " adding: content/llama-dialogue-summarizer/tokenizer_config.json (deflated 67%)\n", " adding: content/llama-dialogue-summarizer/adapter_config.json (deflated 50%)\n" ] } ] }, { "cell_type": "code", "source": [ "from transformers import TextStreamer\n", "model.config.use_cache = True\n", "model.eval()" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "K5mB2ambg0OO", "outputId": "ef3d35e9-3867-4e97-e548-2a0db6d53d1c" }, "execution_count": null, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "PeftModel(\n", " (base_model): LoraModel(\n", " (model): LlamaForCausalLM(\n", " (model): LlamaModel(\n", " (embed_tokens): Embedding(32000, 4096, padding_idx=0)\n", " (layers): ModuleList(\n", " (0-31): 32 x LlamaDecoderLayer(\n", " (self_attn): LlamaAttention(\n", " (q_proj): lora.Linear4bit(\n", " (base_layer): Linear4bit(in_features=4096, out_features=4096, bias=False)\n", " (lora_dropout): ModuleDict(\n", " (default): Dropout(p=0.1, inplace=False)\n", " )\n", " (lora_A): ModuleDict(\n", " (default): Linear(in_features=4096, out_features=16, bias=False)\n", " )\n", " (lora_B): ModuleDict(\n", " (default): Linear(in_features=16, out_features=4096, bias=False)\n", " )\n", " (lora_embedding_A): ParameterDict()\n", " (lora_embedding_B): ParameterDict()\n", " )\n", " (k_proj): lora.Linear4bit(\n", " (base_layer): Linear4bit(in_features=4096, out_features=4096, bias=False)\n", " (lora_dropout): ModuleDict(\n", " (default): Dropout(p=0.1, inplace=False)\n", " )\n", " (lora_A): ModuleDict(\n", " (default): Linear(in_features=4096, out_features=16, bias=False)\n", " )\n", " (lora_B): ModuleDict(\n", " (default): Linear(in_features=16, out_features=4096, bias=False)\n", " )\n", " (lora_embedding_A): ParameterDict()\n", " (lora_embedding_B): ParameterDict()\n", " )\n", " (v_proj): lora.Linear4bit(\n", " (base_layer): Linear4bit(in_features=4096, out_features=4096, bias=False)\n", " (lora_dropout): ModuleDict(\n", " (default): Dropout(p=0.1, inplace=False)\n", " )\n", " (lora_A): ModuleDict(\n", " (default): Linear(in_features=4096, out_features=16, bias=False)\n", " )\n", " (lora_B): ModuleDict(\n", " (default): Linear(in_features=16, out_features=4096, bias=False)\n", " )\n", " (lora_embedding_A): ParameterDict()\n", " (lora_embedding_B): ParameterDict()\n", " )\n", " (o_proj): lora.Linear4bit(\n", " (base_layer): Linear4bit(in_features=4096, out_features=4096, bias=False)\n", " (lora_dropout): ModuleDict(\n", " (default): Dropout(p=0.1, inplace=False)\n", " )\n", " (lora_A): ModuleDict(\n", " (default): Linear(in_features=4096, out_features=16, bias=False)\n", " )\n", " (lora_B): ModuleDict(\n", " (default): Linear(in_features=16, out_features=4096, bias=False)\n", " )\n", " (lora_embedding_A): ParameterDict()\n", " (lora_embedding_B): ParameterDict()\n", " )\n", " (rotary_emb): LlamaRotaryEmbedding()\n", " )\n", " (mlp): LlamaMLP(\n", " (gate_proj): Linear4bit(in_features=4096, out_features=11008, bias=False)\n", " (up_proj): Linear4bit(in_features=4096, out_features=11008, bias=False)\n", " (down_proj): Linear4bit(in_features=11008, out_features=4096, bias=False)\n", " (act_fn): SiLUActivation()\n", " )\n", " (input_layernorm): LlamaRMSNorm()\n", " (post_attention_layernorm): LlamaRMSNorm()\n", " )\n", " )\n", " (norm): LlamaRMSNorm()\n", " )\n", " (lm_head): Linear(in_features=4096, out_features=32000, bias=False)\n", " )\n", " )\n", ")" ] }, "metadata": {}, "execution_count": 21 } ] } ] }