Leetmonkey In Action via Inference

app.py

@@ -33,11 +33,11 @@ def download_model(model_name):
 model_path = download_model(MODEL_NAME)
 llm = Llama(
     model_path=model_path,
-    n_ctx=1024,
+    n_ctx=512,
     n_threads=8,
-    n_gpu_layers=-1,  # Use all available GPU layers
+    n_gpu_layers=1,  # Use all available GPU layers
     verbose=False,
-    n_batch=512,
+    n_batch=1024,
     mlock=True
 )
 logger.info("8-bit model loaded successfully")
@@ -48,11 +48,11 @@ train_dataset = dataset["train"]
 
 # Generation parameters
 generation_kwargs = {
-    "max_tokens": 512,
+    "max_tokens": 256,
     "stop": ["```", "### Instruction:", "### Response:"],
     "echo": False,
-    "temperature": 0.05,
-    "top_k": 10,
+    "temperature": 0.01,
+    "top_k": 5,
     "top_p": 0.9,
     "repeat_penalty": 1.1
 }