New versions with longer context

adapter_config.json

@@ -9,7 +9,7 @@
   "merge_weights": false,
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 8,
+  "r": 16,
   "target_modules": [
     "q_proj",
     "v_proj"

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9aff0d265b6996fbaa2fe3310cd67e1a29fa2515ae9fb2d5dd85d5a9c04c405a
-size 26271757
+oid sha256:03a3ca054813fcd99852f5dba7a606d28f780178cbc8930eb65eff7ae6ce1c42
+size 52486477

generation_config.json

@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 0,
+  "temperature": 1.0,
+  "top_p": 0.9,
+  "top_k": 30,
+  "do_sample": true,
+  "max_new_tokens": 512,
+  "num_beams": 3,
+  "repetition_penalty": 1.1,
+  "no_repeat_ngram_size": 6
+}

training_config.json

@@ -0,0 +1,34 @@
+{
+    "trainer": {
+        "evaluation_strategy": "steps",
+        "per_device_train_batch_size": 4,
+        "per_device_eval_batch_size": 4,
+        "gradient_accumulation_steps": 32,
+        "eval_steps": 150,
+        "save_steps": 150,
+        "logging_steps": 5,
+        "learning_rate": 0.0003,
+        "num_train_epochs": 3,
+        "lr_scheduler_type": "cosine",
+        "warmup_steps": 100,
+        "fp16": true,
+        "bf16": false,
+        "torch_compile": false,
+        "optim": "adamw_torch"
+    },
+    "lora": {
+        "r": 16,
+        "lora_alpha": 16,
+        "lora_dropout": 0.05,
+        "bias": "none",
+        "target_modules": ["q_proj", "v_proj"],
+        "task_type": "CAUSAL_LM"
+    },
+    "load_in_8bit": true,
+    "only_target_loss": false,
+    "model_name": "models/llama-13b-hf",
+    "model_type": "causal",
+    "template_category": "causal_newlines",
+    "max_source_tokens_count": 512,
+    "max_target_tokens_count": 832
+}