{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [], "gpuType": "T4", "include_colab_link": true }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" }, "accelerator": "GPU" }, "cells": [ { "cell_type": "markdown", "metadata": { "id": "view-in-github", "colab_type": "text" }, "source": [ "\"Open" ] }, { "cell_type": "code", "source": [ "# @title # 🤖 AutoQuantize\n", "\n", "# @markdown 🔮 Created by [@zainulabideen](https://huggingface.co/abideen).\n", "\n", "# @markdown Please add HF token to the secrets tab in Google Colab before.\n", "\n", "# @markdown Quantization formats supported: `GGUF`, `AWQ`, `EXL2`, `GPTQ`\n", "\n", "# @markdown ---\n", "\n", "\n", "# @markdown ### 🤗 Hugging Face Hub\n", "\n", "MODEL_ID = \"abideen/Heimer-dpo-TinyLlama-1.1B\" # @param {type:\"string\"}\n", "MODEL_NAME = MODEL_ID.split('/')[-1]\n", "\n", "# Download model\n", "!git lfs install\n", "!git clone https://huggingface.co/{MODEL_ID}\n", "\n", "username = \"abideen\" # @param {type:\"string\"}\n", "token = \"\" # @param {type:\"string\"}\n", "!pip install -q huggingface_hub\n", "from huggingface_hub import create_repo, HfApi\n", "from google.colab import userdata, runtime" ], "metadata": { "id": "fD24jJxq7t3k", "cellView": "form" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# @title # 🛸 GGUF\n", "# @markdown ### ✨ Quantization parameters\n", "\n", "QUANTIZATION_FORMAT = \"q4_k_m\" # @param {type:\"string\"}\n", "QUANTIZATION_METHODS = QUANTIZATION_FORMAT.replace(\" \", \"\").split(\",\")\n", "# Install llama.cpp\n", "!git clone https://github.com/ggerganov/llama.cpp\n", "!cd llama.cpp && git pull && make clean && LLAMA_CUBLAS=1 make\n", "!pip install -r llama.cpp/requirements.txt\n", "\n", "# Convert to fp16\n", "fp16 = f\"{MODEL_NAME}/{MODEL_NAME.lower()}.fp16.bin\"\n", "!python llama.cpp/convert.py {MODEL_NAME} --outtype f16 --outfile {fp16}\n", "\n", "# Quantize the model for each method in the QUANTIZATION_METHODS list\n", "for method in QUANTIZATION_METHODS:\n", " qtype = f\"{MODEL_NAME}/{MODEL_NAME.lower()}.{method.upper()}.gguf\"\n", " !./llama.cpp/quantize {fp16} {qtype} {method}\n", "\n", "# Defined in the secrets tab in Google Colab\n", "hf_token = userdata.get(token)\n", "api = HfApi()\n", "\n", "# Create empty repo\n", "create_repo(\n", " repo_id = f\"{username}/{MODEL_NAME}-GGUF\",\n", " repo_type=\"model\",\n", " exist_ok=True,\n", " token=hf_token\n", ")\n", "\n", "# Upload gguf files\n", "api.upload_folder(\n", " folder_path=MODEL_NAME,\n", " repo_id=f\"{username}/{MODEL_NAME}-GGUF\",\n", " allow_patterns=[\"*.gguf\",\"$.md\"],\n", " token=hf_token\n", ")" ], "metadata": { "id": "NL0yGhbe3EFk", "cellView": "form" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# @title # 🏛️ AWQ\n", "# @markdown ### ✨ Quantization parameters\n", "\n", "Q_GROUP_SIZE = 128 # @param {type:\"integer\"}\n", "ZERO_POINT = True # @param {text:\"boolean\"}\n", "W_BIT = 4 # @param {type:\"integer\"}\n", "VERSION = \"GEMM\" # @param {type:\"string\"}\n", "SAFETENSORS = True # @param {text:\"boolean\"}\n", "\n", "# Install AutoAWQ\n", "!git clone https://github.com/casper-hansen/AutoAWQ\n", "%cd AutoAWQ\n", "!pip install -e .\n", "!pip install git+https://github.com/huggingface/transformers\n", "!pip install zstandard\n", "\n", "from awq import AutoAWQForCausalLM\n", "from transformers import AutoTokenizer\n", "\n", "\n", "quant_path = MODEL_NAME + \"-awq\"\n", "quant_config = { \"zero_point\": ZERO_POINT, \"q_group_size\": Q_GROUP_SIZE, \"w_bit\": W_BIT, \"version\": VERSION }\n", "\n", "# Load model\n", "PATH = \"/content/\" + MODEL_NAME\n", "model = AutoAWQForCausalLM.from_pretrained(PATH, safetensors=SAFETENSORS)\n", "tokenizer = AutoTokenizer.from_pretrained(PATH, trust_remote_code=True)\n", "\n", "# Quantize\n", "model.quantize(tokenizer, quant_config=quant_config)\n", "\n", "# Save quantized model\n", "model.save_quantized(quant_path)\n", "tokenizer.save_pretrained(quant_path)\n", "\n", "# Defined in the secrets tab in Google Colab\n", "hf_token = userdata.get(token)\n", "api = HfApi()\n", "\n", "# Create empty repo\n", "create_repo(\n", " repo_id = f\"{username}/{MODEL_NAME}-AWQ\",\n", " repo_type=\"model\",\n", " exist_ok=True,\n", " token=hf_token\n", ")\n", "\n", "# Upload awq files\n", "api.upload_folder(\n", " folder_path=quant_path,\n", " repo_id=f\"{username}/{MODEL_NAME}-AWQ\",\n", " token=hf_token\n", ")" ], "metadata": { "id": "MyyUO2Fj3WHt", "cellView": "form" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# @title # 🔬 EXL2\n", "# @markdown ### ✨ Quantization parameters\n", "\n", "BPW = 5.0 # @param {type:\"number\"}\n", "\n", "# Install ExLLamaV2\n", "!git clone https://github.com/turboderp/exllamav2\n", "!pip install -e exllamav2\n", "\n", "!mv {MODEL_NAME} base_model\n", "!rm base_mode/*.bin\n", "\n", "# Download dataset\n", "!wget https://huggingface.co/datasets/wikitext/resolve/9a9e482b5987f9d25b3a9b2883fc6cc9fd8071b3/wikitext-103-v1/wikitext-test.parquet\n", "\n", "# Quantize model\n", "!mkdir quant\n", "!python exllamav2/convert.py \\\n", " -i base_model \\\n", " -o quant \\\n", " -c wikitext-test.parquet \\\n", " -b {BPW}\n", "\n", "# Copy files\n", "!rm -rf quant/out_tensor\n", "!rsync -av --exclude='*.safetensors' --exclude='.*' ./base_model/ ./quant/\n", "\n", "# Defined in the secrets tab in Google Colab\n", "hf_token = userdata.get(token)\n", "api = HfApi()\n", "\n", "# Create empty repo\n", "create_repo(\n", " repo_id = f\"{username}/{MODEL_NAME}-{BPW:.1f}bpw-exl2\",\n", " repo_type=\"model\",\n", " exist_ok=True,\n", " token=hf_token\n", ")\n", "\n", "# Upload exl2 files\n", "api.upload_folder(\n", " folder_path=quant,\n", " repo_id=f\"{username}/{MODEL_NAME}-{BPW:.1f}bpw-exl2\",\n", " token=hf_token\n", ")" ], "metadata": { "id": "ZC9Nsr9u5WhN", "cellView": "form" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# @title # 📝 GPTQ\n", "# @markdown ### ✨ Quantization parameters\n", "\n", "BITS = 4 # @param {type:\"integer\"}\n", "GROUP_SIZE = 128 # @param {type:\"integer\"}\n", "DAMP_PERCENT = 0.01 # @param {type:\"number\"}\n", "\n", "!BUILD_CUDA_EXT=0 pip install -q auto-gptq transformers\n", "import random\n", "from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig\n", "from datasets import load_dataset\n", "import torch\n", "from transformers import AutoTokenizer\n", "out_dir = MODEL_ID + \"-GPTQ\"\n", "\n", "# Load quantize config, model and tokenizer\n", "quantize_config = BaseQuantizeConfig(\n", " bits=BITS,\n", " group_size=GROUP_SIZE,\n", " damp_percent=DAMP_PERCENT,\n", " desc_act=False,\n", ")\n", "PATH = \"/content/\" + MODEL_NAME\n", "model = AutoGPTQForCausalLM.from_pretrained(PATH, quantize_config)\n", "tokenizer = AutoTokenizer.from_pretrained(PATH)\n", "\n", "# Load data and tokenize examples\n", "n_samples = 1024\n", "data = load_dataset(\"allenai/c4\", data_files=\"en/c4-train.00001-of-01024.json.gz\", split=f\"train[:{n_samples*5}]\")\n", "tokenized_data = tokenizer(\"\\n\\n\".join(data['text']), return_tensors='pt')\n", "\n", "# Format tokenized examples\n", "examples_ids = []\n", "for _ in range(n_samples):\n", " i = random.randint(0, tokenized_data.input_ids.shape[1] - tokenizer.model_max_length - 1)\n", " j = i + tokenizer.model_max_length\n", " input_ids = tokenized_data.input_ids[:, i:j]\n", " attention_mask = torch.ones_like(input_ids)\n", " examples_ids.append({'input_ids': input_ids, 'attention_mask': attention_mask})\n", "\n", "# Quantize with GPTQ\n", "model.quantize(\n", " examples_ids,\n", " batch_size=1,\n", " use_triton=True,\n", ")\n", "\n", "# Save model and tokenizer\n", "model.save_quantized(out_dir, use_safetensors=True)\n", "tokenizer.save_pretrained(out_dir)\n", "\n", "# Defined in the secrets tab in Google Colab\n", "hf_token = userdata.get(token)\n", "api = HfApi()\n", "\n", "# Create empty repo\n", "create_repo(\n", " repo_id = f\"{username}/{MODEL_NAME}-GPTQ\",\n", " repo_type=\"model\",\n", " exist_ok=True,\n", " token=hf_token\n", ")\n", "\n", "# Upload gptq files\n", "api.upload_folder(\n", " folder_path=out_dir,\n", " repo_id=f\"{username}/{MODEL_NAME}-GPTQ\",\n", " token=hf_token\n", ")\n" ], "metadata": { "id": "OE_R3AXG5Y-F", "cellView": "form" }, "execution_count": null, "outputs": [] } ] }