phi-3.5-mini.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "if '__file__' not in globals():\n",
    "    __file__, __name__ = globals()['__vsc_ipynb_file__'], '__ipynb__'\n",
    "    import types, sys; sys.modules['__ipynb__'] = types.ModuleType('__ipynb__')\n",
    "    from IPython.core.magic import register_cell_magic\n",
    "    @register_cell_magic\n",
    "    def skip_if(flag, cell): exec(cell, globals())if flag and not eval(flag) else print('Cell skipped...')\n",
    "\n",
    "import sys, os\n",
    "if os.path.abspath('.') not in sys.path: sys.path.append(os.path.abspath('.'))\n",
    "\n",
    "import os, huggingface_hub # !pip install huggingface_hub[hf_transfer]\n",
    "huggingface_hub.login(token = os.environ.get('HF_TOKEN'), add_to_git_credential=True)\n",
    "\n",
    "import inspect\n",
    "from pathlib import Path\n",
    "from tqdm import tqdm\n",
    "from glob import glob\n",
    "import numpy as np; np.set_printoptions(precision=8, suppress=True); np.random.seed(42)\n",
    "\n",
    "class text_color:\n",
    "    black,red,green,yellow,blue,magenta,cyan,white,gray = [*range(30,38), 90] # fgclr,  [*range(90,98), ''] # light-fgclr\n",
    "    bold, italic, underline, strike = 1, 3, 4, 9  # attrs supported on vscode notebook.\n",
    "    def __init__(self, fg,bg=0,attr=0):\n",
    "        attr = f'{attr};' if attr > 0 else ''\n",
    "        bg = f'{bg+10};' if bg > 0 else ''\n",
    "        self.clr = f'\\33[{attr}{bg}{fg}m'\n",
    "\n",
    "    def __ror__(self, obj): return self.clr + str(obj) + '\\33[0m'\n",
    "    @staticmethod\n",
    "    def all(): return (text_color(clr) for clr in [*range(30,38), 90])\n",
    "\n",
    "black,red,green,yellow,blue,magenta,cyan,white,gray = text_color.all()\n",
    "\n",
    "class cout:\n",
    "    def __ror__(self, obj): print(f'[{inspect.stack()[1].lineno}] {str(obj)}')\n",
    "    def __call__(self, *args, **kwds): print(f'[{inspect.stack()[1].lineno+1}]', *args, **kwds)\n",
    "out = cout()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### load weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://colab.research.google.com/drive/1uskRfCesambdQrhGp2s5wx6zGwSUR0Sh\n",
    "# https://huggingface.co/microsoft/Phi-3.5-mini-instruct/blob/main/config.json\n",
    "# https://github.com/microsoft/Phi-3CookBook\n",
    "# !pip install -q sentencepiece\n",
    "\n",
    "import torch, peft, transformers\n",
    "\n",
    "model_id = \"microsoft/Phi-3.5-mini-instruct\" # 128k, 7.7GB\n",
    "model_id = \"outputs_unslot/merged-model\" # ft.\n",
    "\n",
    "model = transformers.AutoModelForCausalLM.from_pretrained(\n",
    "    model_id,\n",
    "    device_map=\"auto\",\n",
    "    torch_dtype=torch.bfloat16,\n",
    "    trust_remote_code=True,\n",
    "    low_cpu_mem_usage=True,\n",
    "    attn_implementation=\"flash_attention_2\",\n",
    ")\n",
    "\n",
    "# model_id = \"outputs_unslot/model/lora\"\n",
    "# model = peft.AutoPeftModelForCausalLM.from_pretrained(\n",
    "#     model_id, device_map=\"cuda\",\n",
    "#     torch_dtype=\"auto\",\n",
    "#     trust_remote_code=True,\n",
    "#     low_cpu_mem_usage=True,\n",
    "#     attn_implementation=\"flash_attention_2\")\n",
    "\n",
    "model.config.use_cache = True\n",
    "model.eval()\n",
    "print(model)\n",
    "\n",
    "tokenizer = transformers.AutoTokenizer.from_pretrained(model_id)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import display, Markdown, clear_output\n",
    "\n",
    "class CallbackTextStreamer(transformers.TextStreamer):\n",
    "    def __init__(self, markdown=False, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.markdown = markdown\n",
    "        self.ans = \"\"\n",
    "\n",
    "    def on_finalized_text(self, text: str, stream_end: bool = False):\n",
    "        super().on_finalized_text(text, stream_end)\n",
    "        self.ans += text\n",
    "        if stream_end:\n",
    "            self.finally_markdown()\n",
    "\n",
    "    def finally_markdown(self):\n",
    "        if self.markdown:\n",
    "            clear_output(wait=False)\n",
    "            display(Markdown(self.ans))\n",
    "\n",
    "def generate(input_text, system_prompt=\"\",max_length=512, stream=True):\n",
    "\n",
    "    if isinstance(input_text, list):\n",
    "        messages = input_text\n",
    "    else:\n",
    "        if not system_prompt:\n",
    "            system_prompt = \"You are a friendly and helpful assistant\"\n",
    "        messages = [\n",
    "            { \"role\": \"system\", \"content\": system_prompt,},\n",
    "            {\"role\": \"user\", \"content\": input_text},\n",
    "        ]\n",
    "\n",
    "    streamer = CallbackTextStreamer(tokenizer=tokenizer, skip_prompt=True) if stream else None\n",
    "\n",
    "    prompt = tokenizer.apply_chat_template(messages,\n",
    "                                           tokenize=False,\n",
    "                                           add_generation_prompt=True)\n",
    "\n",
    "    inputs = tokenizer.encode(prompt, add_special_tokens=True, return_tensors=\"pt\").to(\"cuda\")\n",
    "    outputs = model.generate(input_ids=inputs.to(model.device),\n",
    "                             max_new_tokens=max_length,\n",
    "                             do_sample=True,\n",
    "                             temperature=0.1,\n",
    "                             top_k=50,\n",
    "                             streamer=streamer,\n",
    "                             )\n",
    "    # text = ''.join([t for t in streamer])\n",
    "\n",
    "    text = tokenizer.decode(outputs[0, len(inputs[0]):],skip_special_tokens=True, clean_up_tokenization_spaces=True)\n",
    "    return text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "So the answer: 16\n",
      "\n",
      "Here's the breakdown:\n",
      "\n",
      "1. Kaylee has sold 12 boxes of lemon biscuits.\n",
      "2. She has sold 5 boxes of chocolate biscuits.\n",
      "3. She has sold 4 boxes of oatmeal biscuits.\n",
      "4. Total boxes sold so far: 12 (lemon) + 5 (chocolate) + 4 (oatmeal) = 21 boxes.\n",
      "5. Kaylee needs to sell a total of 33 boxes.\n",
      "6. Boxes still needed to be sold: 33 (total) - 21 (sold) = 12 boxes.\n",
      "\n",
      "Therefore, Kaylee needs to sell 12 more boxes of biscuits.<|end|>\n"
     ]
    }
   ],
   "source": [
    "# reformat answer for better cot training\n",
    "#\n",
    "system = \"\"\"Following is dataset to fine tuning llm with cot concept.\n",
    "To make more good dataset, reformat assistant's cot answer as numbering. And lastly the answer with leading words 'So the answer: '. Answer the reformatted string only.\"\"\"\n",
    "\n",
    "content = '''\n",
    "{\"type\": \"cot\", \"messages\": [{\"role\": \"user\", \"content\": \"Kaylee needs to sell 33 boxes of biscuits. So far, she has sold 12 boxes of lemon biscuits to her aunt, 5 boxes of chocolate biscuits to her mother, and 4 boxes of oatmeal biscuits to a neighbor. How many more boxes of biscuits does Kaylee need to sell? Let's think first. Chain of thought:\"}, {\"role\": \"assistant\", \"content\": \"To solve this problem, we need to find out how many boxes of biscuits Kaylee has sold so far and then subtract that number from the total number of boxes she needs to sell. We can do this by adding up the number of boxes she has sold and then subtracting that sum from the total number of boxes she needs to sell.\"}], \"source\": \"open-orca\"}\n",
    "'''\n",
    "# 33 - (12 + 5 + 4) = 12\n",
    "\n",
    "content = '''\\\n",
    "user query: \"Kaylee needs to sell 33 boxes of biscuits. So far, she has sold 12 boxes of lemon biscuits to her aunt, 5 boxes of chocolate biscuits to her mother, and 4 boxes of oatmeal biscuits to a neighbor. How many more boxes of biscuits does Kaylee need to sell? Let's think first. Chain of thought:\"\n",
    "\n",
    "assistant answer: \"To solve this problem, we need to find out how many boxes of biscuits Kaylee has sold so far and then subtract that number from the total number of boxes she needs to sell. We can do this by adding up the number of boxes she has sold and then subtracting that sum from the total number of boxes she needs to sell.\"\n",
    "'''\n",
    "gen = generate(content, system, max_length=512)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Markdown( generate('Write a detailed analogy between mathematics and a lighthouse.',\n",
    "         system_prompt=\"You are Phi, a small language model trained by Microsoft. Write out your reasoning step-by-step to be sure you get the right answers!\",\n",
    "         max_length=1024))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate('Write a short email to Sam Altman giving reasons to open source GPT-4',\n",
    "         system_prompt=\"You are Phi-3, a large language model trained by Microsoft. Write out your reasoning step-by-step to be sure you get the right answers!\",\n",
    "         max_length=512, stream=True);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate('''```python\n",
    "def detect_prime(n):\n",
    "   \"\"\"\n",
    "   detect if a number is a prime number or not. return True or False\n",
    "   \"\"\"''',\n",
    "   system_prompt=\"You are a genius python coder, please think carefully and write the following code:\");\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gen = generate('Answer the following question by reasoning step by step. The cafeteria had 23 apples. If they used 20 for lunch, and bought 6 more, how many apple do they have?',\n",
    "         system_prompt=\"You are Phi-3, a large language model trained by Microsoft. Write out your reasoning step-by-step to be sure you get the right answers!\",\n",
    "         max_length=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gen = generate(\"Weng earns $12 an hour for babysitting. Yesterday, she just did 50 minutes of babysitting. How much did she earn?\",\n",
    "         system_prompt=\"Write out your reasoning step-by-step to be sure you get the right answers!\",\n",
    "         max_length=512)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### function call"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datasets, re, json\n",
    "\n",
    "ds_test = datasets.load_dataset(\"json\", data_files=\"xlam-dataset-60k-qwen2-test.jsonl\")['train']\n",
    "\n",
    "system_prompt = \"Write out your reasoning step-by-step to be sure you get the right answers!\"\n",
    "\n",
    "\n",
    "for i, sample in enumerate(ds_test):\n",
    "    message = sample[\"messages\"]\n",
    "    user_content = message[0]['content']\n",
    "    # message.insert(0, dict(role='system', content=system_prompt))\n",
    "\n",
    "    ans = message[-1][\"content\"]\n",
    "    gen = generate(message[:-1], stream=False)\n",
    "    gen = gen.replace('```json', '').replace('```', '')\n",
    "\n",
    "    true,false = True,False\n",
    "    gen = json.dumps(eval(gen), indent=3)\n",
    "    ans = json.dumps(eval(ans), indent=3)\n",
    "\n",
    "    if gen != ans:\n",
    "        # print(f'{i}.query:', user_content[-200:]|gray)\n",
    "        print('gen:', gen|red)\n",
    "        print('ans:', ans|green)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### from https://deepgram.com/learn/chain-of-thought-prompting-guide"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Roger starts with 5 tennis balls.\n",
      "\n",
      "He buys 2 more cans, and each can has 3 tennis balls. So, the number of tennis balls in the cans is:\n",
      "2 cans * 3 tennis balls per can = 6 tennis balls\n",
      "\n",
      "Now, we add the tennis balls he already had to the ones he bought:\n",
      "5 tennis balls + 6 tennis balls = 11 tennis balls\n",
      "\n",
      "Roger now has a total of 11 tennis balls.<|end|>\n",
      "answer: 11\n"
     ]
    }
   ],
   "source": [
    "prompts = [\n",
    "(\"Roger has 5 tennis balls. He buys 2 more cans of tennis balls. Each can has 3 tennis balls. How many tennis balls does he have now?\", 11),\n",
    "(\"Yes or no: Would a pear sink in water?\",None),\n",
    "(\"How would you bring me something that isn’t a fruit?\",None),\n",
    "(\"How many keystrokes are needed to type the numbers from 1 to 500?\", 1392),\n",
    "(\"The concert was scheduled to be on 06/01/1943, but was delayed by one day to today. What is the date 10 days ago in MM/DD/YYYY?\", \"05/23/1943\"),\n",
    "(\"Take the last letters of the words in 'Lady Gaga' and concatenate them.\", 'ya'),\n",
    "(\"Sammy wanted to go to where the people were. Where might he go?\",None),\n",
    "(\"Is the following sentence plausible? 'Joao Moutinho caught the screen pass in the NFC championship.'\", 'not plausible'),\n",
    "(\"A coin is heads up. Maybelle flips the coin. Shalonda does not flip the coin. Is the coin still heads up?\", \"No\"),\n",
    "('Answer the following question by reasoning step by step. The cafeteria had 23 apples. If they used 20 for lunch, and bought 6 more, how many apple do they have?', 9)\n",
    "]\n",
    "\n",
    "line = 2\n",
    "generate(prompts[line-2][0],\n",
    "         system_prompt=\"Write out your reasoning step-by-step to be sure you get the right answers!\",\n",
    "         max_length=512)\n",
    "\n",
    "if prompts[line-2][1]:\n",
    "    print('answer:', prompts[line-2][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "react_prompt = \"\"\"\\\n",
    "Assistant is a large language model trained by Microsoft.\n",
    "\n",
    "Assistant is designed to be able to assist with a wide range of tasks, from answering simple questions to providing in-depth explanations and discussions on a wide range of topics. As a language model, Assistant is able to generate human-like text based on the input it receives, allowing it to engage in natural-sounding conversations and provide responses that are coherent and relevant to the topic at hand.\n",
    "\n",
    "Assistant is constantly learning and improving, and its capabilities are constantly evolving. It is able to process and understand large amounts of text, and can use this knowledge to provide accurate and informative responses to a wide range of questions. Additionally, Assistant is able to generate its own text based on the input it receives, allowing it to engage in discussions and provide explanations and descriptions on a wide range of topics.\n",
    "\n",
    "Overall, Assistant is a powerful tool that can help with a wide range of tasks and provide valuable insights and information on a wide range of topics. Whether you need help with a specific question or just want to have a conversation about a particular topic, Assistant is here to assist.\n",
    "\n",
    "TOOLS:\n",
    "------\n",
    "\n",
    "Assistant has access to the following tools:\n",
    "\n",
    "wikipedia_search - searches the wikipedia database for the answer\\n\n",
    "web_search - searches the web for the answer\\n\n",
    "calculator - calculates the answer to the question\\n\n",
    "weather_api - gets the weather for the location\\n\n",
    "\n",
    "\n",
    "To use a tool, please use the following format:\n",
    "\n",
    "```\n",
    "Thought: Do I need to use a tool? Yes\n",
    "Action: the action to take, should be one of [wikipedia_search, web_search, calculator, weather_api]\n",
    "Action Input: the input to the action\n",
    "Observation: the result of the action\n",
    "```\n",
    "\n",
    "When you have a response to say to the Human, or if you do not need to use a tool, you MUST use the format:\n",
    "\n",
    "```\n",
    "Thought: Do I need to use a tool? No\n",
    "Final Answer: [your response here]\n",
    "```\n",
    "\n",
    "Begin!\n",
    "\n",
    "\n",
    "New input: {query}\n",
    "\"\"\" #{agent_scratchpad}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gen = generate(react_prompt.format(query=\"What is the latest AI news today?\"), max_length=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate(react_prompt.format(query=\"오늘 날씨 어때?\"), max_length=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate(\"사과와 귤중 어떤게 더 맛있어?\", max_length=1024)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ollama"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "from IPython.display import display, Markdown\n",
    "from multimethod import multimethod\n",
    "_ollama = openai.OpenAI(base_url='http://localhost:11434/v1')\n",
    "\n",
    "ollama_model_id = 'phi3.5:3.8b-mini-instruct-q8_0'\n",
    "ollama_model_id = 'jjkim76/phi3.5-fc:Q8_0'\n",
    "\n",
    "def generate(input_text, system_prompt=None, max_length=1024):\n",
    "    if system_prompt is None:\n",
    "        system_prompt = \"\"\"Write out your reasoning step-by-step to be sure you get the right answers!\"\"\"\n",
    "\n",
    "    messages = input_text\n",
    "    if not isinstance(messages, list):\n",
    "        messages = [dict(role=\"system\", content=system_prompt), dict(role=\"user\", content=input_text)]\n",
    "\n",
    "    response = _ollama.chat.completions.create(\n",
    "        model=ollama_model_id,\n",
    "        messages=messages,\n",
    "        temperature = 0.01, top_p = 0.01,\n",
    "        max_tokens=max_length,\n",
    "        # do_sample=True, temperature=0.1, top_k=50,\n",
    "    )\n",
    "    text = response.choices[0].message.content\n",
    "    # print(input_text[-200:]|gray)\n",
    "    # display(Markdown(text))\n",
    "    return text\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fcv3-2",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}