Leetmonkey In Action via Inference

app.py

@@ -33,10 +33,12 @@ def download_model(model_name):
 model_path = download_model(MODEL_NAME)
 llm = Llama(
     model_path=model_path,
-    n_ctx=2048,
+    n_ctx=1024,
     n_threads=4,
     n_gpu_layers=-1,  # Use all available GPU layers
-    verbose=False
+    verbose=False,
+    n_batch=512,
+    mlock=True
 )
 logger.info("8-bit model loaded successfully")
 
@@ -46,12 +48,12 @@ train_dataset = dataset["train"]
 
 # Generation parameters
 generation_kwargs = {
-    "max_tokens": 2048,
+    "max_tokens": 1024,
     "stop": ["```", "### Instruction:", "### Response:"],
     "echo": False,
-    "temperature": 0.2,
-    "top_k": 50,
-    "top_p": 0.95,
+    "temperature": 0.1,
+    "top_k": 20,
+    "top_p": 0.9,
     "repeat_penalty": 1.1
 }