Leetmonkey In Action via Inference

app.py

@@ -34,7 +34,7 @@ model_path = download_model(MODEL_NAME)
 llm = Llama(
     model_path=model_path,
     n_ctx=1024,
-    n_threads=4,
+    n_threads=8,
     n_gpu_layers=-1,  # Use all available GPU layers
     verbose=False,
     n_batch=512,
@@ -48,11 +48,11 @@ train_dataset = dataset["train"]
 
 # Generation parameters
 generation_kwargs = {
-    "max_tokens": 1024,
+    "max_tokens": 512,
     "stop": ["```", "### Instruction:", "### Response:"],
     "echo": False,
-    "temperature": 0.1,
-    "top_k": 20,
+    "temperature": 0.05,
+    "top_k": 10,
     "top_p": 0.9,
     "repeat_penalty": 1.1
 }