add device=cuda; initialize tokenizer and model outside of function call; add inputs to device

app.py

@@ -3,6 +3,11 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
 import gradio as gr
 import spaces
 
+device = "cuda" # for GPU usage or "cpu" for CPU usage
+
+tokenizer = AutoTokenizer.from_pretrained("PatronusAI/Llama-3-Patronus-Lynx-8B-Instruct")
+model = AutoModelForCausalLM.from_pretrained("PatronusAI/Llama-3-Patronus-Lynx-8B-Instruct", torch_dtype=torch.float16, device_map="auto")
+
 PROMPT = """
 Given the following QUESTION, DOCUMENT and ANSWER you must analyze the provided answer and determine whether it is faithful to the contents of the DOCUMENT. The ANSWER must not offer new information beyond the context provided in the DOCUMENT. The ANSWER also must not contradict information provided in the DOCUMENT. Output your final verdict by strictly following this format: "PASS" if the answer is faithful to the DOCUMENT and "FAIL" if the answer is not faithful to the DOCUMENT. Show your reasoning.
 
@@ -23,13 +28,10 @@ ANSWER:
 Your output should be in JSON FORMAT with the keys "REASONING" and "SCORE":
 {{"REASONING": <your reasoning as bullet points>, "SCORE": <your final score>}}
 """
-
-
+@spaces.GPU()
 def model_call(question, document, answer):
     NEW_FORMAT = PROMPT.format(question=question, document=document, answer=answer)
-    tokenizer = AutoTokenizer.from_pretrained("PatronusAI/Llama-3-Patronus-Lynx-8B-Instruct")
-    model = AutoModelForCausalLM.from_pretrained("PatronusAI/Llama-3-Patronus-Lynx-8B-Instruct", torch_dtype=torch.float16, device_map="auto")
-    inputs = tokenizer(NEW_FORMAT, return_tensors="pt")
+    inputs = tokenizer(NEW_FORMAT, return_tensors="pt").to(device)
     model.generate(inputs.input_ids)
     generated_text = tokenizer.decode(inputs.input_ids[0])
     print(generated_text)