Update README.md

README.md

@@ -1,3 +1,114 @@
----
-license: llama3
----
+---
+license: llama3
+---
+
+github [Web-UI](https://github.com/moseshu/llama2-chat/tree/main/webui)
+
+![image/png](https://cdn-uploads.huggingface.co/production/uploads/62f4c7172f63f904a0c61ba3/NGBrnC1eaeTcDE_A3niyn.png)
+
+```python
+from transformers import GenerationConfig, LlamaForCausalLM, LlamaTokenizer,AutoTokenizer,AutoModelForCausalLM,MistralForCausalLM
+import torch
+model = AutoModelForCausalLM.from_pretrained(model_id,torch_dtype=torch.bfloat16,device_map="auto",)
+from transformers import GenerationConfig, LlamaForCausalLM, LlamaTokenizer,AutoTokenizer,AutoModelForCausalLM,MistralForCausalLM
+import torch
+
+
+model_id = Moses25/Llama-3-8B-chat-32K
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+
+
+mistral_template="{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}"
+
+llama3_template="{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}"
+
+def chat_format(conversation:list,tokenizer,chat_type="mistral"):
+    system_prompt = "You are a helpful, respectful and honest assistant.Help humman as much as you can."
+    ap = [{"role":"system","content":system_prompt}] + conversation
+    if chat_type=='mistral':
+        id = tokenizer.apply_chat_template(ap,chat_template=mistral_template,tokenize=False)
+    elif chat_type=='llama3':
+        id = tokenizer.apply_chat_template(ap,chat_template=llama3_template,tokenize=False)
+        id = id.rstrip("<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n")
+    return id
+
+user_chat=[{"role":"user","content":"In a basket, there are 20 oranges, 60 apples, and 40 bananas. If 15 pears were added, and half of the oranges were removed, what would be the new ratio of oranges to apples, bananas, and pears combined within the basket?"}]
+text = chat_format(user_chat,tokenizer,'llama3')
+def predict(content_prompt):
+    inputs = tokenizer(content_prompt,return_tensors="pt",add_special_tokens=True)
+    input_ids = inputs["input_ids"].to("cuda:0")
+    # print(f"input length:{len(input_ids[0])}")
+    with torch.no_grad():
+        generation_output = model.generate(
+                    input_ids=input_ids,
+                    #generation_config=generation_config,
+                    return_dict_in_generate=True,
+                    output_scores=True,
+                    max_new_tokens=2048,
+                    top_p=0.9,
+                    num_beams=1,
+                    do_sample=True,
+                    repetition_penalty=1.0,
+                    eos_token_id=tokenizer.eos_token_id,
+                    pad_token_id=tokenizer.pad_token_id,
+                )
+        s = generation_output.sequences[0]
+        output = tokenizer.decode(s,skip_special_tokens=True)
+        output1 = output.split("<|eot_id|>")[-1].strip()
+        # print(output1)
+    return output1
+
+predict(text)
+output："""Let's break down the steps to find the new ratio of oranges to apples, bananas, and pears combined:
+Calculate the total number of fruits initially in the basket: Oranges: 20 Apples: 60 Bananas: 40 Total Fruits = 20 + 60 + 40 = 120
+Add 15 pears: Total Fruits after adding pears = 120 + 15 = 135
+Remove half of the oranges: Oranges remaining = 20 / 2 = 10
+Calculate the total number of fruits remaining in the basket after removing half of the oranges: Total Remaining Fruits = 10 (oranges) + 60 (apples) + 40 (bananas) + 15 (pears) = 125
+Find the ratio of oranges to apples, bananas, and pears combined: Ratio of Oranges to (Apples, Bananas, Pears) Combined = Oranges / (Apples + Bananas + Pears) = 10 / (60 + 40 + 15) = 10 / 115
+So, the new ratio of oranges to apples, bananas, and pears combined within the basket is 10:115.
+However, I should note that the actual fruit distribution in your basket may vary depending on how you decide to count and categorize the fruits. The example calculation provides a theoretical ratio based on the initial quantities mentioned."""
+```
+
+
+## vLLM server
+```shell
+#llama3-chat-template.jinja file is chat-template above 'llama3-template'
+model_path = Llama-3-8B-chat-32K
+python  -m vllm.entrypoints.openai.api_server --model=$model_path \
+        --trust-remote-code --host 0.0.0.0  --port 7777 \
+        --gpu-memory-utilization 0.8 \
+        --max-model-len 8192 --chat-template llama3-chat-template.jinja \
+        --tensor-parallel-size 1 --served-model-name chatbot
+```
+
+```python
+from openai import OpenAI
+# Set OpenAI's API key and API base to use vLLM's API server.
+openai_api_key = "EMPTY"
+openai_api_base = "http://localhost:7777/v1"
+
+client = OpenAI(
+    api_key=openai_api_key,
+    base_url=openai_api_base,
+)
+call_args = {
+        'temperature': 0.7,
+        'top_p': 0.9,
+        'top_k': 40,
+        'max_tokens': 2048, # output-len
+        'presence_penalty': 1.0,
+        'frequency_penalty': 0.0,
+        "repetition_penalty":1.0,
+        "stop":["<|eot_id|>","<|end_of_text|>"],
+    }
+chat_response = client.chat.completions.create(
+    model="chatbot",
+    messages=[
+        {"role": "user", "content": "你好"},
+    ],
+    extra_body=call_args
+)
+print("Chat response:", chat_response)
+
+
+```