# import gradio as gr # from huggingface_hub import InferenceClient # """ # For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference # """ # client = InferenceClient("meta-llama/Meta-Llama-3-8B-Instruct") # ## None type # def respond( # message: str, # history: list[tuple[str, str]], # This will not be used # system_message: str, # max_tokens: int, # temperature: float, # top_p: float, # ): # messages = [{"role": "system", "content": system_message}] # # Append only the latest user message # messages.append({"role": "user", "content": message}) # response = "" # try: # # Generate response from the model # for message in client.chat_completion( # messages, # max_tokens=max_tokens, # stream=True, # temperature=temperature, # top_p=top_p, # ): # if message.choices[0].delta.content is not None: # token = message.choices[0].delta.content # response += token # yield response # except Exception as e: # yield f"An error occurred: {e}" # ], # ) # if __name__ == "__main__": # demo.launch() ##Running smothly CHATBOT # import gradio as gr # from huggingface_hub import InferenceClient # """ # For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference # """ # client = InferenceClient("meta-llama/Meta-Llama-3-8B-Instruct") # def respond( # message: str, # history: list[tuple[str, str]], # This will not be used # system_message: str, # max_tokens: int, # temperature: float, # top_p: float, # ): # # Build the messages list # messages = [{"role": "system", "content": system_message}] # messages.append({"role": "user", "content": message}) # response = "" # try: # # Generate response from the model # for msg in client.chat_completion( # messages=messages, # max_tokens=max_tokens, # stream=True, # temperature=temperature, # top_p=top_p, # ): # if msg.choices[0].delta.content is not None: # token = msg.choices[0].delta.content # response += token # yield response # except Exception as e: # yield f"An error occurred: {e}" # """ # For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface # """ # demo = gr.ChatInterface( # respond, # additional_inputs=[ # gr.Textbox(value="You are a friendly Chatbot.", label="System message"), # gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"), # gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"), # gr.Slider( # minimum=0.1, # maximum=1.0, # value=0.95, # step=0.05, # label="Top-p (nucleus sampling)", # ), # ], # ) # if __name__ == "__main__": # demo.launch() ####03 3.1 8b # import os # import time # import spaces # import torch # from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig # import gradio as gr # from threading import Thread # MODEL_LIST = ["meta-llama/Meta-Llama-3.1-8B-Instruct"] # HF_TOKEN = os.environ.get("HF_API_TOKEN",None) # print(HF_TOKEN,"######$$$$$$$$$$$$$$$") # MODEL = os.environ.get("MODEL_ID","meta-llama/Meta-Llama-3.1-8B-Instruct") # TITLE = "

Meta-Llama3.1-8B

" # PLACEHOLDER = """ #
#

Hi! How can I help you today?

#
# """ # CSS = """ # .duplicate-button { # margin: auto !important; # color: white !important; # background: black !important; # border-radius: 100vh !important; # } # h3 { # text-align: center; # } # """ # device = "cuda" # for GPU usage or "cpu" for CPU usage # quantization_config = BitsAndBytesConfig( # load_in_4bit=True, # bnb_4bit_compute_dtype=torch.bfloat16, # bnb_4bit_use_double_quant=True, # bnb_4bit_quant_type= "nf4") # tokenizer = AutoTokenizer.from_pretrained(MODEL) # model = AutoModelForCausalLM.from_pretrained( # MODEL, # torch_dtype=torch.bfloat16, # device_map="auto", # quantization_config=quantization_config) # @spaces.GPU() # def stream_chat( # message: str, # history: list, # system_prompt: str, # temperature: float = 0.8, # max_new_tokens: int = 1024, # top_p: float = 1.0, # top_k: int = 20, # penalty: float = 1.2, # ): # print(f'message: {message}') # print(f'history: {history}') # conversation = [ # {"role": "system", "content": system_prompt} # ] # for prompt, answer in history: # conversation.extend([ # {"role": "user", "content": prompt}, # {"role": "assistant", "content": answer}, # ]) # conversation.append({"role": "user", "content": message}) # input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt").to(model.device) # streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True) # generate_kwargs = dict( # input_ids=input_ids, # max_new_tokens = max_new_tokens, # do_sample = False if temperature == 0 else True, # top_p = top_p, # top_k = top_k, # temperature = temperature, # repetition_penalty=penalty, # eos_token_id=[128001,128008,128009], # streamer=streamer, # ) # with torch.no_grad(): # thread = Thread(target=model.generate, kwargs=generate_kwargs) # thread.start() # buffer = "" # for new_text in streamer: # buffer += new_text # yield buffer # chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER) # with gr.Blocks(css=CSS, theme="soft") as demo: # gr.HTML(TITLE) # gr.DuplicateButton(value="Duplicate Space for private use", elem_classes="duplicate-button") # gr.ChatInterface( # fn=stream_chat, # chatbot=chatbot, # fill_height=True, # additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False), # additional_inputs=[ # gr.Textbox( # value="You are a helpful assistant", # label="System Prompt", # render=False, # ), # gr.Slider( # minimum=0, # maximum=1, # step=0.1, # value=0.8, # label="Temperature", # render=False, # ), # gr.Slider( # minimum=128, # maximum=8192, # step=1, # value=1024, # label="Max new tokens", # render=False, # ), # gr.Slider( # minimum=0.0, # maximum=1.0, # step=0.1, # value=1.0, # label="top_p", # render=False, # ), # gr.Slider( # minimum=1, # maximum=20, # step=1, # value=20, # label="top_k", # render=False, # ), # gr.Slider( # minimum=0.0, # maximum=2.0, # step=0.1, # value=1.2, # label="Repetition penalty", # render=False, # ), # ], # examples=[ # ["Help me study vocabulary: write a sentence for me to fill in the blank, and I'll try to pick the correct option."], # ["What are 5 creative things I could do with my kids' art? I don't want to throw them away, but it's also so much clutter."], # ["Tell me a random fun fact about the Roman Empire."], # ["Show me a code snippet of a website's sticky header in CSS and JavaScript."], # ], # cache_examples=False, # ) # if __name__ == "__main__": # demo.launch() ###########new clientkey 04 ruunng chlrhah # import os # import time # import spaces # import torch # from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer # import gradio as gr # from threading import Thread # MODEL = "THUDM/LongWriter-llama3.1-8b" # TITLE = "

AreaX LLC-llama3.1-8b

" # PLACEHOLDER = """ #
#

Hi! I'm AreaX AI Agent, capable of generating 10,000+ words. How can I assist you today?

#
# """ # CSS = """ # .duplicate-button { # margin: auto !important; # color: white !important; # background: black !important; # border-radius: 100vh !important; # } # h3 { # text-align: center; # } # """ # device = "cuda" if torch.cuda.is_available() else "cpu" # tokenizer = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=True) # model = AutoModelForCausalLM.from_pretrained(MODEL, torch_dtype=torch.bfloat16, trust_remote_code=True, device_map="auto") # model = model.eval() # @spaces.GPU() # def stream_chat( # message: str, # history: list, # system_prompt: str, # temperature: float = 0.5, # max_new_tokens: int = 32768, # top_p: float = 1.0, # top_k: int = 50, # ): # print(f'message: {message}') # print(f'history: {history}') # full_prompt = f"<>\n{system_prompt}\n<>\n\n" # for prompt, answer in history: # full_prompt += f"[INST]{prompt}[/INST]{answer}" # full_prompt += f"[INST]{message}[/INST]" # inputs = tokenizer(full_prompt, truncation=False, return_tensors="pt").to(device) # context_length = inputs.input_ids.shape[-1] # streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True) # generate_kwargs = dict( # inputs=inputs.input_ids, # max_new_tokens=max_new_tokens, # do_sample=True, # top_p=top_p, # top_k=top_k, # temperature=temperature, # num_beams=1, # streamer=streamer, # ) # thread = Thread(target=model.generate, kwargs=generate_kwargs) # thread.start() # buffer = "" # for new_text in streamer: # buffer += new_text # yield buffer # chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER) # with gr.Blocks(css=CSS, theme="soft") as demo: # gr.HTML(TITLE) # gr.DuplicateButton(value="Duplicate Space for private use", elem_classes="duplicate-button") # gr.ChatInterface( # fn=stream_chat, # chatbot=chatbot, # fill_height=True, # additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False), # additional_inputs=[ # gr.Textbox( # value="You are a helpful assistant capable of generating long-form content.", # label="System Prompt", # render=False, # ), # gr.Slider( # minimum=0, # maximum=1, # step=0.1, # value=0.5, # label="Temperature", # render=False, # ), # gr.Slider( # minimum=1024, # maximum=32768, # step=1024, # value=32768, # label="Max new tokens", # render=False, # ), # gr.Slider( # minimum=0.0, # maximum=1.0, # step=0.1, # value=1.0, # label="Top p", # render=False, # ), # gr.Slider( # minimum=1, # maximum=100, # step=1, # value=50, # label="Top k", # render=False, # ), # ], # # examples=[ # # ["Write a 5000-word comprehensive guide on machine learning for beginners."], # # ["Create a detailed 3000-word business plan for a sustainable energy startup."], # # ["Compose a 2000-word short story set in a futuristic underwater city."], # # ["Develop a 4000-word research proposal on the potential effects of climate change on global food security."], # # ], # # cache_examples=False, # ) # if __name__ == "__main__": # demo.launch() # ###OCT04 LLAMA3.2 Vision Model from transformers import MllamaForConditionalGeneration, AutoProcessor, TextIteratorStreamer from PIL import Image import requests import torch from threading import Thread import gradio as gr from gradio import FileData import time import os import spaces from huggingface_hub import login login(token=os.getenv("HF_API_TOKEN")) # ckpt = "meta-llama/Llama-3.2-11B-Vision-Instruct" # model = MllamaForConditionalGeneration.from_pretrained(ckpt, # torch_dtype=torch.bfloat16).to("cuda") # processor = AutoProcessor.from_pretrained(ckpt) # @spaces.GPU # def bot_streaming(message, history, max_new_tokens=250): # txt = message["text"] # ext_buffer = f"{txt}" # messages= [] # images = [] # for i, msg in enumerate(history): # if isinstance(msg[0], tuple): # messages.append({"role": "user", "content": [{"type": "text", "text": history[i+1][0]}, {"type": "image"}]}) # messages.append({"role": "assistant", "content": [{"type": "text", "text": history[i+1][1]}]}) # images.append(Image.open(msg[0][0]).convert("RGB")) # elif isinstance(history[i-1], tuple) and isinstance(msg[0], str): # # messages are already handled # pass # elif isinstance(history[i-1][0], str) and isinstance(msg[0], str): # text only turn # messages.append({"role": "user", "content": [{"type": "text", "text": msg[0]}]}) # messages.append({"role": "assistant", "content": [{"type": "text", "text": msg[1]}]}) # # add current message # if len(message["files"]) == 1: # if isinstance(message["files"][0], str): # examples # image = Image.open(message["files"][0]).convert("RGB") # else: # regular input # image = Image.open(message["files"][0]["path"]).convert("RGB") # images.append(image) # messages.append({"role": "user", "content": [{"type": "text", "text": txt}, {"type": "image"}]}) # else: # messages.append({"role": "user", "content": [{"type": "text", "text": txt}]}) # texts = processor.apply_chat_template(messages, add_generation_prompt=True) # if images == []: # inputs = processor(text=texts, return_tensors="pt").to("cuda") # else: # inputs = processor(text=texts, images=images, return_tensors="pt").to("cuda") # streamer = TextIteratorStreamer(processor, skip_special_tokens=True, skip_prompt=True) # generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=max_new_tokens) # generated_text = "" # thread = Thread(target=model.generate, kwargs=generation_kwargs) # thread.start() # buffer = "" # for new_text in streamer: # buffer += new_text # generated_text_without_prompt = buffer # time.sleep(0.01) # yield buffer # demo = gr.ChatInterface(fn=bot_streaming, title="Multimodal Llama", # textbox=gr.MultimodalTextbox(), # additional_inputs = [gr.Slider( # minimum=10, # maximum=500, # value=250, # step=10, # label="Maximum number of new tokens to generate", # ) # ], # cache_examples=False, # description="Try Multimodal Llama by Meta with transformers in this demo. Upload an image, and start chatting about it, or simply try one of the examples below. To learn more about Llama Vision, visit [our blog post](https://huggingface.co/blog/llama32). ", # stop_btn="Stop Generation", # fill_height=True, # multimodal=True) # demo.launch(debug=True,live=True) ckpt = "meta-llama/Llama-3.2-11B-Vision-Instruct" model = MllamaForConditionalGeneration.from_pretrained(ckpt, torch_dtype=torch.bfloat16).to("cuda") processor = AutoProcessor.from_pretrained(ckpt) @spaces.GPU def bot_streaming(message, history, max_new_tokens=1000): txt = message["text"] ext_buffer = f"{txt}" messages = [] images = [] # Process history messages for i, msg in enumerate(history): if isinstance(msg[0], tuple): messages.append({"role": "user", "content": [{"type": "text", "text": history[i+1][0]}, {"type": "image"}]}) messages.append({"role": "assistant", "content": [{"type": "text", "text": history[i+1][1]}]}) images.append(Image.open(msg[0][0]).convert("RGB")) elif isinstance(history[i-1], tuple) and isinstance(msg[0], str): pass # Previous messages already handled elif isinstance(history[i-1][0], str) and isinstance(msg[0], str): # Text-only turn messages.append({"role": "user", "content": [{"type": "text", "text": msg[0]}]}) messages.append({"role": "assistant", "content": [{"type": "text", "text": msg[1]}]}) # Add current message if len(message["files"]) == 1: if isinstance(message["files"][0], str): # Example images image = Image.open(message["files"][0]).convert("RGB") else: # Regular input image = Image.open(message["files"][0]["path"]).convert("RGB") images.append(image) messages.append({"role": "user", "content": [{"type": "text", "text": txt}, {"type": "image"}]}) else: messages.append({"role": "user", "content": [{"type": "text", "text": txt}]}) # Prepare input for the model texts = processor.apply_chat_template(messages, add_generation_prompt=True) if not images: inputs = processor(text=texts, return_tensors="pt").to("cuda") else: inputs = processor(text=texts, images=images, return_tensors="pt").to("cuda") streamer = TextIteratorStreamer(processor, skip_special_tokens=True, skip_prompt=True) generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=max_new_tokens) generated_text = "" # Start text generation in a separate thread thread = Thread(target=model.generate, kwargs=generation_kwargs) thread.start() buffer = "" for new_text in streamer: buffer += new_text time.sleep(0.01) # Small delay to simulate streaming yield buffer # Gradio interface setup demo = gr.ChatInterface( fn=bot_streaming, title="AreaX-Llama3.2-11B-Vision", textbox=gr.MultimodalTextbox(), additional_inputs=[ gr.Slider( minimum=10, maximum=500, value=250, step=10, label="Maximum number of new tokens to generate", ) ], cache_examples=False, description="Try AreaX Llama3.2-11B Vision Model by Meta with transformers in this demo. Upload an image, and start chatting about it, or simply type your question.", stop_btn="Stop Generation", fill_height=True, multimodal=True ) demo.launch(debug=True,share=True)