Deployed model

app.py

@@ -290,318 +290,318 @@
 ###########new clientkey 04 ruunng chlrhah 
 
 
-import os
-import time
-import spaces
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
-import gradio as gr
-from threading import Thread
+# import os
+# import time
+# import spaces
+# import torch
+# from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+# import gradio as gr
+# from threading import Thread
 
-MODEL = "THUDM/LongWriter-llama3.1-8b"
-
-TITLE = "<h1><center>AreaX LLC-llama3.1-8b</center></h1>"
-
-PLACEHOLDER = """
-<center>
-<p>Hi! I'm AreaX AI Agent, capable of generating 10,000+ words. How can I assist you today?</p>
-</center>
-"""
-
-CSS = """
-.duplicate-button {
-    margin: auto !important;
-    color: white !important;
-    background: black !important;
-    border-radius: 100vh !important;
-}
-h3 {
-    text-align: center;
-}
-"""
-
-device = "cuda" if torch.cuda.is_available() else "cpu"
-
-tokenizer = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=True)
-model = AutoModelForCausalLM.from_pretrained(MODEL, torch_dtype=torch.bfloat16, trust_remote_code=True, device_map="auto")
-model = model.eval()
-
-@spaces.GPU()
-def stream_chat(
-    message: str,
-    history: list,
-    system_prompt: str,
-    temperature: float = 0.5,
-    max_new_tokens: int = 32768,
-    top_p: float = 1.0,
-    top_k: int = 50,
-):
-    print(f'message: {message}')
-    print(f'history: {history}')
-
-    full_prompt = f"<<SYS>>\n{system_prompt}\n<</SYS>>\n\n"
-    for prompt, answer in history:
-        full_prompt += f"[INST]{prompt}[/INST]{answer}"
-    full_prompt += f"[INST]{message}[/INST]"
-
-    inputs = tokenizer(full_prompt, truncation=False, return_tensors="pt").to(device)
-    context_length = inputs.input_ids.shape[-1]
-
-    streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True)
-
-    generate_kwargs = dict(
-        inputs=inputs.input_ids,
-        max_new_tokens=max_new_tokens,
-        do_sample=True,
-        top_p=top_p,
-        top_k=top_k,
-        temperature=temperature,
-        num_beams=1,
-        streamer=streamer,
-    )
-
-    thread = Thread(target=model.generate, kwargs=generate_kwargs)
-    thread.start()
+# MODEL = "THUDM/LongWriter-llama3.1-8b"
 
-    buffer = ""
-    for new_text in streamer:
-        buffer += new_text
-        yield buffer
+# TITLE = "<h1><center>AreaX LLC-llama3.1-8b</center></h1>"
 
-chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER)
-
-with gr.Blocks(css=CSS, theme="soft") as demo:
-    gr.HTML(TITLE)
-    gr.DuplicateButton(value="Duplicate Space for private use", elem_classes="duplicate-button")
-    gr.ChatInterface(
-        fn=stream_chat,
-        chatbot=chatbot,
-        fill_height=True,
-        additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False),
-        additional_inputs=[
-            gr.Textbox(
-                value="You are a helpful assistant capable of generating long-form content.",
-                label="System Prompt",
-                render=False,
-            ),
-            gr.Slider(
-                minimum=0,
-                maximum=1,
-                step=0.1,
-                value=0.5,
-                label="Temperature",
-                render=False,
-            ),
-            gr.Slider(
-                minimum=1024,
-                maximum=32768,
-                step=1024,
-                value=32768,
-                label="Max new tokens",
-                render=False,
-            ),
-            gr.Slider(
-                minimum=0.0,
-                maximum=1.0,
-                step=0.1,
-                value=1.0,
-                label="Top p",
-                render=False,
-            ),
-            gr.Slider(
-                minimum=1,
-                maximum=100,
-                step=1,
-                value=50,
-                label="Top k",
-                render=False,
-            ),
-        ],
-        # examples=[
-        #     ["Write a 5000-word comprehensive guide on machine learning for beginners."],
-        #     ["Create a detailed 3000-word business plan for a sustainable energy startup."],
-        #     ["Compose a 2000-word short story set in a futuristic underwater city."],
-        #     ["Develop a 4000-word research proposal on the potential effects of climate change on global food security."],
-        # ],
-        # cache_examples=False,
-    )
-
-if __name__ == "__main__":
-    demo.launch()
+# PLACEHOLDER = """
+# <center>
+# <p>Hi! I'm AreaX AI Agent, capable of generating 10,000+ words. How can I assist you today?</p>
+# </center>
+# """
 
+# CSS = """
+# .duplicate-button {
+#     margin: auto !important;
+#     color: white !important;
+#     background: black !important;
+#     border-radius: 100vh !important;
+# }
+# h3 {
+#     text-align: center;
+# }
+# """
 
+# device = "cuda" if torch.cuda.is_available() else "cpu"
 
-# ###OCT04 LLAMA3.2 Vision Model
-# from transformers import MllamaForConditionalGeneration, AutoProcessor, TextIteratorStreamer
-# from PIL import Image
-# import requests
-# import torch
-# from threading import Thread
-# import gradio as gr
-# from gradio import FileData
-# import time
-# import os
-# import spaces
-# from huggingface_hub import login
-# login(token=os.getenv("HF_API_TOKEN"))
-# # ckpt = "meta-llama/Llama-3.2-11B-Vision-Instruct"
-# # model = MllamaForConditionalGeneration.from_pretrained(ckpt,
-# #     torch_dtype=torch.bfloat16).to("cuda")
-# # processor = AutoProcessor.from_pretrained(ckpt)
+# tokenizer = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=True)
+# model = AutoModelForCausalLM.from_pretrained(MODEL, torch_dtype=torch.bfloat16, trust_remote_code=True, device_map="auto")
+# model = model.eval()
 
+# @spaces.GPU()
+# def stream_chat(
+#     message: str,
+#     history: list,
+#     system_prompt: str,
+#     temperature: float = 0.5,
+#     max_new_tokens: int = 32768,
+#     top_p: float = 1.0,
+#     top_k: int = 50,
+# ):
+#     print(f'message: {message}')
+#     print(f'history: {history}')
+
+#     full_prompt = f"<<SYS>>\n{system_prompt}\n<</SYS>>\n\n"
+#     for prompt, answer in history:
+#         full_prompt += f"[INST]{prompt}[/INST]{answer}"
+#     full_prompt += f"[INST]{message}[/INST]"
+
+#     inputs = tokenizer(full_prompt, truncation=False, return_tensors="pt").to(device)
+#     context_length = inputs.input_ids.shape[-1]
+
+#     streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True)
+
+#     generate_kwargs = dict(
+#         inputs=inputs.input_ids,
+#         max_new_tokens=max_new_tokens,
+#         do_sample=True,
+#         top_p=top_p,
+#         top_k=top_k,
+#         temperature=temperature,
+#         num_beams=1,
+#         streamer=streamer,
+#     )
+
+#     thread = Thread(target=model.generate, kwargs=generate_kwargs)
+#     thread.start()
+
+#     buffer = ""
+#     for new_text in streamer:
+#         buffer += new_text
+#         yield buffer
+
+# chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER)
+
+# with gr.Blocks(css=CSS, theme="soft") as demo:
+#     gr.HTML(TITLE)
+#     gr.DuplicateButton(value="Duplicate Space for private use", elem_classes="duplicate-button")
+#     gr.ChatInterface(
+#         fn=stream_chat,
+#         chatbot=chatbot,
+#         fill_height=True,
+#         additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False),
+#         additional_inputs=[
+#             gr.Textbox(
+#                 value="You are a helpful assistant capable of generating long-form content.",
+#                 label="System Prompt",
+#                 render=False,
+#             ),
+#             gr.Slider(
+#                 minimum=0,
+#                 maximum=1,
+#                 step=0.1,
+#                 value=0.5,
+#                 label="Temperature",
+#                 render=False,
+#             ),
+#             gr.Slider(
+#                 minimum=1024,
+#                 maximum=32768,
+#                 step=1024,
+#                 value=32768,
+#                 label="Max new tokens",
+#                 render=False,
+#             ),
+#             gr.Slider(
+#                 minimum=0.0,
+#                 maximum=1.0,
+#                 step=0.1,
+#                 value=1.0,
+#                 label="Top p",
+#                 render=False,
+#             ),
+#             gr.Slider(
+#                 minimum=1,
+#                 maximum=100,
+#                 step=1,
+#                 value=50,
+#                 label="Top k",
+#                 render=False,
+#             ),
+#         ],
+#         # examples=[
+#         #     ["Write a 5000-word comprehensive guide on machine learning for beginners."],
+#         #     ["Create a detailed 3000-word business plan for a sustainable energy startup."],
+#         #     ["Compose a 2000-word short story set in a futuristic underwater city."],
+#         #     ["Develop a 4000-word research proposal on the potential effects of climate change on global food security."],
+#         # ],
+#         # cache_examples=False,
+#     )
+
+# if __name__ == "__main__":
+#     demo.launch()
 
-# # @spaces.GPU
-# # def bot_streaming(message, history, max_new_tokens=250):
-    
-# #     txt = message["text"]
-# #     ext_buffer = f"{txt}"
-    
-# #     messages= [] 
-# #     images = []
-    
 
-# #     for i, msg in enumerate(history): 
-# #         if isinstance(msg[0], tuple):
-# #             messages.append({"role": "user", "content": [{"type": "text", "text": history[i+1][0]}, {"type": "image"}]})
-# #             messages.append({"role": "assistant", "content": [{"type": "text", "text": history[i+1][1]}]})
-# #             images.append(Image.open(msg[0][0]).convert("RGB"))
-# #         elif isinstance(history[i-1], tuple) and isinstance(msg[0], str):
-# #             # messages are already handled
-# #             pass
-# #         elif isinstance(history[i-1][0], str) and isinstance(msg[0], str): # text only turn
-# #             messages.append({"role": "user", "content": [{"type": "text", "text": msg[0]}]})
-# #             messages.append({"role": "assistant", "content": [{"type": "text", "text": msg[1]}]})
-
-# #     # add current message
-# #     if len(message["files"]) == 1:
-        
-# #         if isinstance(message["files"][0], str): # examples
-# #             image = Image.open(message["files"][0]).convert("RGB")
-# #         else: # regular input
-# #             image = Image.open(message["files"][0]["path"]).convert("RGB")
-# #         images.append(image)
-# #         messages.append({"role": "user", "content": [{"type": "text", "text": txt}, {"type": "image"}]})
-# #     else:
-# #         messages.append({"role": "user", "content": [{"type": "text", "text": txt}]})
-
-
-# #     texts = processor.apply_chat_template(messages, add_generation_prompt=True)
-
-# #     if images == []:
-# #         inputs = processor(text=texts, return_tensors="pt").to("cuda")
-# #     else:
-# #         inputs = processor(text=texts, images=images, return_tensors="pt").to("cuda")
-# #     streamer = TextIteratorStreamer(processor, skip_special_tokens=True, skip_prompt=True)
-
-# #     generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=max_new_tokens)
-# #     generated_text = ""
-    
-# #     thread = Thread(target=model.generate, kwargs=generation_kwargs)
-# #     thread.start()
-# #     buffer = ""
-    
-# #     for new_text in streamer:
-# #         buffer += new_text
-# #         generated_text_without_prompt = buffer
-# #         time.sleep(0.01)
-# #         yield buffer
-
-
-# # demo = gr.ChatInterface(fn=bot_streaming, title="Multimodal Llama",
-# #       textbox=gr.MultimodalTextbox(), 
-# #       additional_inputs = [gr.Slider(
-# #               minimum=10,
-# #               maximum=500,
-# #               value=250,
-# #               step=10,
-# #               label="Maximum number of new tokens to generate",
-# #           )
-# #         ],
-# #       cache_examples=False,
-# #       description="Try Multimodal Llama by Meta with transformers in this demo. Upload an image, and start chatting about it, or simply try one of the examples below. To learn more about Llama Vision, visit [our blog post](https://huggingface.co/blog/llama32). ",
-# #       stop_btn="Stop Generation", 
-# #       fill_height=True,
-# #     multimodal=True)
-    
-# # demo.launch(debug=True,live=True)
 
+# ###OCT04 LLAMA3.2 Vision Model
+from transformers import MllamaForConditionalGeneration, AutoProcessor, TextIteratorStreamer
+from PIL import Image
+import requests
+import torch
+from threading import Thread
+import gradio as gr
+from gradio import FileData
+import time
+import os
+import spaces
+from huggingface_hub import login
+login(token=os.getenv("HF_API_TOKEN"))
 # ckpt = "meta-llama/Llama-3.2-11B-Vision-Instruct"
-# model = MllamaForConditionalGeneration.from_pretrained(ckpt, torch_dtype=torch.bfloat16).to("cuda")
+# model = MllamaForConditionalGeneration.from_pretrained(ckpt,
+#     torch_dtype=torch.bfloat16).to("cuda")
 # processor = AutoProcessor.from_pretrained(ckpt)
 
+
 # @spaces.GPU
-# def bot_streaming(message, history, max_new_tokens=1000):
+# def bot_streaming(message, history, max_new_tokens=250):
+    
 #     txt = message["text"]
 #     ext_buffer = f"{txt}"
     
-#     messages = [] 
+#     messages= [] 
 #     images = []
     
-#     # Process history messages
+
 #     for i, msg in enumerate(history): 
 #         if isinstance(msg[0], tuple):
 #             messages.append({"role": "user", "content": [{"type": "text", "text": history[i+1][0]}, {"type": "image"}]})
 #             messages.append({"role": "assistant", "content": [{"type": "text", "text": history[i+1][1]}]})
 #             images.append(Image.open(msg[0][0]).convert("RGB"))
 #         elif isinstance(history[i-1], tuple) and isinstance(msg[0], str):
-#             pass  # Previous messages already handled
-#         elif isinstance(history[i-1][0], str) and isinstance(msg[0], str):  # Text-only turn
+#             # messages are already handled
+#             pass
+#         elif isinstance(history[i-1][0], str) and isinstance(msg[0], str): # text only turn
 #             messages.append({"role": "user", "content": [{"type": "text", "text": msg[0]}]})
 #             messages.append({"role": "assistant", "content": [{"type": "text", "text": msg[1]}]})
 
-#     # Add current message
+#     # add current message
 #     if len(message["files"]) == 1:
-#         if isinstance(message["files"][0], str):  # Example images
+        
+#         if isinstance(message["files"][0], str): # examples
 #             image = Image.open(message["files"][0]).convert("RGB")
-#         else:  # Regular input
+#         else: # regular input
 #             image = Image.open(message["files"][0]["path"]).convert("RGB")
 #         images.append(image)
 #         messages.append({"role": "user", "content": [{"type": "text", "text": txt}, {"type": "image"}]})
 #     else:
 #         messages.append({"role": "user", "content": [{"type": "text", "text": txt}]})
 
-#     # Prepare input for the model
+
 #     texts = processor.apply_chat_template(messages, add_generation_prompt=True)
 
-#     if not images:
+#     if images == []:
 #         inputs = processor(text=texts, return_tensors="pt").to("cuda")
 #     else:
 #         inputs = processor(text=texts, images=images, return_tensors="pt").to("cuda")
-    
 #     streamer = TextIteratorStreamer(processor, skip_special_tokens=True, skip_prompt=True)
 
 #     generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=max_new_tokens)
 #     generated_text = ""
     
-#     # Start text generation in a separate thread
 #     thread = Thread(target=model.generate, kwargs=generation_kwargs)
 #     thread.start()
 #     buffer = ""
     
 #     for new_text in streamer:
 #         buffer += new_text
-#         time.sleep(0.01)  # Small delay to simulate streaming
+#         generated_text_without_prompt = buffer
+#         time.sleep(0.01)
 #         yield buffer
 
-# # Gradio interface setup
-# demo = gr.ChatInterface(
-#     fn=bot_streaming, 
-#     title="AreaX-Llama3.2-11B-Vision", 
-#     textbox=gr.MultimodalTextbox(), 
-#     additional_inputs=[
-#         gr.Slider(
-#             minimum=10,
-#             maximum=500,
-#             value=250,
-#             step=10,
-#             label="Maximum number of new tokens to generate",
-#         )
-#     ],
-#     cache_examples=False,
-#     description="Try AreaX Llama3.2-11B Vision Model by Meta with transformers in this demo. Upload an image, and start chatting about it, or simply type your question.",
-#     stop_btn="Stop Generation", 
-#     fill_height=True,
-#     multimodal=True
-# )
 
-# demo.launch(debug=True,share=True)
+# demo = gr.ChatInterface(fn=bot_streaming, title="Multimodal Llama",
+#       textbox=gr.MultimodalTextbox(), 
+#       additional_inputs = [gr.Slider(
+#               minimum=10,
+#               maximum=500,
+#               value=250,
+#               step=10,
+#               label="Maximum number of new tokens to generate",
+#           )
+#         ],
+#       cache_examples=False,
+#       description="Try Multimodal Llama by Meta with transformers in this demo. Upload an image, and start chatting about it, or simply try one of the examples below. To learn more about Llama Vision, visit [our blog post](https://huggingface.co/blog/llama32). ",
+#       stop_btn="Stop Generation", 
+#       fill_height=True,
+#     multimodal=True)
+    
+# demo.launch(debug=True,live=True)
+
+ckpt = "meta-llama/Llama-3.2-11B-Vision-Instruct"
+model = MllamaForConditionalGeneration.from_pretrained(ckpt, torch_dtype=torch.bfloat16).to("cuda")
+processor = AutoProcessor.from_pretrained(ckpt)
+
+@spaces.GPU
+def bot_streaming(message, history, max_new_tokens=1000):
+    txt = message["text"]
+    ext_buffer = f"{txt}"
+    
+    messages = [] 
+    images = []
+    
+    # Process history messages
+    for i, msg in enumerate(history): 
+        if isinstance(msg[0], tuple):
+            messages.append({"role": "user", "content": [{"type": "text", "text": history[i+1][0]}, {"type": "image"}]})
+            messages.append({"role": "assistant", "content": [{"type": "text", "text": history[i+1][1]}]})
+            images.append(Image.open(msg[0][0]).convert("RGB"))
+        elif isinstance(history[i-1], tuple) and isinstance(msg[0], str):
+            pass  # Previous messages already handled
+        elif isinstance(history[i-1][0], str) and isinstance(msg[0], str):  # Text-only turn
+            messages.append({"role": "user", "content": [{"type": "text", "text": msg[0]}]})
+            messages.append({"role": "assistant", "content": [{"type": "text", "text": msg[1]}]})
+
+    # Add current message
+    if len(message["files"]) == 1:
+        if isinstance(message["files"][0], str):  # Example images
+            image = Image.open(message["files"][0]).convert("RGB")
+        else:  # Regular input
+            image = Image.open(message["files"][0]["path"]).convert("RGB")
+        images.append(image)
+        messages.append({"role": "user", "content": [{"type": "text", "text": txt}, {"type": "image"}]})
+    else:
+        messages.append({"role": "user", "content": [{"type": "text", "text": txt}]})
+
+    # Prepare input for the model
+    texts = processor.apply_chat_template(messages, add_generation_prompt=True)
+
+    if not images:
+        inputs = processor(text=texts, return_tensors="pt").to("cuda")
+    else:
+        inputs = processor(text=texts, images=images, return_tensors="pt").to("cuda")
+    
+    streamer = TextIteratorStreamer(processor, skip_special_tokens=True, skip_prompt=True)
+
+    generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=max_new_tokens)
+    generated_text = ""
+    
+    # Start text generation in a separate thread
+    thread = Thread(target=model.generate, kwargs=generation_kwargs)
+    thread.start()
+    buffer = ""
+    
+    for new_text in streamer:
+        buffer += new_text
+        time.sleep(0.01)  # Small delay to simulate streaming
+        yield buffer
+
+# Gradio interface setup
+demo = gr.ChatInterface(
+    fn=bot_streaming, 
+    title="AreaX-Llama3.2-11B-Vision", 
+    textbox=gr.MultimodalTextbox(), 
+    additional_inputs=[
+        gr.Slider(
+            minimum=10,
+            maximum=500,
+            value=250,
+            step=10,
+            label="Maximum number of new tokens to generate",
+        )
+    ],
+    cache_examples=False,
+    description="Try AreaX Llama3.2-11B Vision Model by Meta with transformers in this demo. Upload an image, and start chatting about it, or simply type your question.",
+    stop_btn="Stop Generation", 
+    fill_height=True,
+    multimodal=True
+)
+
+demo.launch(debug=True,share=True)