import random
import gradio as gr
from ctransformers import AutoModelForCausalLM

local_path = r"itaca_mistral7b_qlora_4bit-unsloth.Q4_K_M.gguf"

model = AutoModelForCausalLM.from_pretrained(local_path, model_file="itaca_mistral7b_qlora_4bit-unsloth.Q4_K_M.gguf", model_type="mistral", local_files_only=True
                                             # , gpu_layers=50

                                             )


def predict(message, history):
    outputs = model(message)

    return outputs


iface = gr.ChatInterface(predict)
iface.launch()