prep gguf

app.py

@@ -1,13 +1,9 @@
-"""Template Demo for IBM Granite Hugging Face spaces."""
-
 from collections.abc import Iterator
 from datetime import datetime
 from pathlib import Path
 from threading import Thread
 
 import gradio as gr
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
 
 from themes.research_monochrome import theme
 
@@ -16,7 +12,7 @@ today_date = datetime.today().strftime("%B %-d, %Y")  # noqa: DTZ002
 SYS_PROMPT = f"""Knowledge Cutoff Date: April 2024.
 Today's Date: {today_date}.
 You are Granite, developed by IBM. You are a helpful AI assistant"""
-TITLE = "IBM Granite 3.1 8b Instruct"
+TITLE = "IBM Granite 3.1 8b Instruct from local GGUF server"
 DESCRIPTION = """
 <p>Granite 3.1 8b instruct is an open-source LLM supporting a 128k context window. Start with one of the sample prompts
 or enter your own. Keep in mind that AI can occasionally make mistakes.
@@ -32,12 +28,10 @@ TOP_P = 0.85
 TOP_K = 50
 REPETITION_PENALTY = 1.05
 
-#model = AutoModelForCausalLM.from_pretrained(
-#    "ibm-granite/granite-3.1-8b-instruct", torch_dtype=torch.float16, device_map="auto"
-#)
-#tokenizer = AutoTokenizer.from_pretrained("ibm-granite/granite-3.1-8b-instruct")
-#tokenizer.use_default_system_prompt = False
+# download GGUF into local directory
 
+# chmod llama-server
+# start llama-server
 
 def generate(
     message: str,
@@ -88,7 +82,6 @@ def generate(
 
 
 css_file_path = Path(Path(__file__).parent / "app.css")
-head_file_path = Path(Path(__file__).parent / "app_head.html")
 
 # advanced settings (displayed in Accordion)
 temperature_slider = gr.Slider(