run.py

import gradio as gr
import random
import time

endpoint_url = "https://ko60a2m26ylqgri6.us-east-1.aws.endpoints.huggingface.cloud"


# Streaming Client
client = InferenceClient(endpoint_url, token=hf_token)

# generation parameter
gen_kwargs = dict(
    max_new_tokens=1024,
    top_k=30,
    top_p=0.9,
    temperature=0.2,
    repetition_penalty=1.05, #1.02
    stop=["\nUser:", "<|endoftext|>", "</s>"],
)
# prompt
# prompt = "What can you do in Nuremberg, Germany? Give me 3 Tips"



with gr.Blocks() as demo:
    chatbot = gr.Chatbot()
    msg = gr.Textbox()
    clear = gr.Button("Clear")

    def user(user_message, history):
        return "", history + [[user_message, None]]

    def bot(history):
        bot_message = client.text_generation(history, stream=True, details=True, **gen_kwargs)
        history[-1][1] = ""
        for character in bot_message:
            history[-1][1] += character
            yield history

    msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then(
        bot, chatbot, chatbot
    )
    clear.click(lambda: None, None, chatbot, queue=False)

if __name__ == "__main__":
    demo.launch()