model.py

from typing import Iterator



model_id = 'theohlong/baichuan2_13b-GGML'

from huggingface_hub import snapshot_download,hf_hub_download

snapshot_download(model_id, local_dir="./", revesion="f416539ad77964d20452f38eaa7a18abf0672eb8b2e58773af3c20039a9d93f1")
hf_hub_download(repo_id="baichuan-inc/Baichuan-13B-Chat-4bits",local_dir="./", filename="tokenizer.model")
from llama_cpp import Llama
llm = Llama(model_path="./ggml-model-q4_1.bin", n_ctx=4096,seed=-1)

def run(message: str,
        chat_history: list[tuple[str, str]],
        system_prompt: str,
        max_new_tokens: int = 1024,
        temperature: float = 0.3,
        top_p: float = 0.85,
        top_k: int = 5) -> Iterator[str]:
    history = []
    print(chat_history)
    result=""
    for i in chat_history:
        history.append({"role": "user", "content": i[0]})
        history.append({"role": "assistant", "content": i[1]})
    print(history)
    history.append({"role": "user", "content": message})
    for response in llm.create_chat_completion(history,stop=["</s>"],stream=True,max_tokens=-1,temperature=temperature,top_k=top_k,top_p=top_p,repeat_penalty=1.1):
        if "content" in response["choices"][0]["delta"]:
            result = result + response["choices"][0]["delta"]["content"]
            yield result