uncommented device=cuda

app.py

@@ -5,7 +5,7 @@ import gradio as gr
 import spaces
 
 HF_TOKEN = os.environ.get("HF_TOKEN", None)
-# device = "cuda" # for GPU usage or "cpu" for CPU usage
+device = "cuda" # for GPU usage or "cpu" for CPU usage
 
 tokenizer = AutoTokenizer.from_pretrained("PatronusAI/Llama-3-Patronus-Lynx-8B-Instruct")
 model = AutoModelForCausalLM.from_pretrained("PatronusAI/Llama-3-Patronus-Lynx-8B-Instruct", torch_dtype=torch.float16, device_map="auto")
@@ -34,7 +34,7 @@ Your output should be in JSON FORMAT with the keys "REASONING" and "SCORE":
 def model_call(question, document, answer):
     NEW_FORMAT = PROMPT.format(question=question, document=document, answer=answer)
     inputs = tokenizer(NEW_FORMAT, return_tensors="pt")
-    input_ids = inputs.input_ids.to(model.device)
+    input_ids = inputs.input_ids.to(device)
     attention_mask = inputs.attention_mask
     generate_kwargs = dict(
         input_ids=input_ids,