app.py

# Created by Keivalya Pandya
# using https://huggingface.co/docs/api-inference/tasks/chat-completion

import gradio as gr
from huggingface_hub import InferenceClient

"""
For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
"""
client = InferenceClient("microsoft/Phi-3-mini-4k-instruct")
# client = InferenceClient("meta-llama/Llama-3.1-8B-Instruct")

# def yodaify_text(text):
#     """
#     Convert the assistant's response to Yoda-style speech.
#     """
#     words = text.split()
#     if len(words) > 4:
#         return f"{' '.join(words[2:])}, {words[0]} {words[1]}"
#     return f"{' '.join(words[::-1])}"

def respond(
    message,
    history: list[tuple[str, str]],
    # system_message,
    # max_tokens,
    # temperature,
    # top_p,
):
    messages = [{"role": "assistant", "content": "You are a helpful assistant who always talks like Yoda from Star Wars."}]

    for val in history:
        if val[0]:
            messages.append({"role": "user", "content": val[0]})
        if val[1]:
            messages.append({"role": "assistant", "content": val[1]})

    messages.append({"role": "user", "content": message})

    response = ""

    for message in client.chat_completion(
        messages,
        max_tokens=128,
        stream=True,
        temperature=0.3,
        top_p=0.8,
    ):
        token = message.choices[0].delta.content

        response += token
        # Modify assistant response to Yoda-style speech
        # yoda_response = yodaify_text(response)
        # yield yoda_response
        yield response


"""
For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
"""
demo = gr.ChatInterface(
    respond,
    # additional_inputs=[
    #     gr.Textbox(value="You are Yoda from Star Wars.", label="System message"),
    #     gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
    #     gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
    #     gr.Slider(
    #         minimum=0.1,
    #         maximum=1.0,
    #         value=0.95,
    #         step=0.05,
    #         label="Top-p (nucleus sampling)",
    #     ),
    # ],
)

if __name__ == "__main__":
    demo.launch()