Upload folder using huggingface_hub

README.md

@@ -1,7 +1,7 @@
 ---
 base_model:
-- OmnicromsBrain/Eros_Scribe-7b
-- MrRobotoAI/Nuit-4
+- MrRobotoAI/Hathor-2
+- MrRobotoAI/Hathor-6
 library_name: transformers
 tags:
 - mergekit
@@ -20,8 +20,8 @@ This model was merged using the SLERP merge method.
 ### Models Merged
 
 The following models were included in the merge:
-* [OmnicromsBrain/Eros_Scribe-7b](https://huggingface.co/OmnicromsBrain/Eros_Scribe-7b)
-* [MrRobotoAI/Nuit-4](https://huggingface.co/MrRobotoAI/Nuit-4)
+* [MrRobotoAI/Hathor-2](https://huggingface.co/MrRobotoAI/Hathor-2)
+* [MrRobotoAI/Hathor-6](https://huggingface.co/MrRobotoAI/Hathor-6)
 
 ### Configuration
 
@@ -30,12 +30,12 @@ The following YAML configuration was used to produce this model:
 ```yaml
 slices:
   - sources:
-      - model: MrRobotoAI/Nuit-4
+      - model: MrRobotoAI/Hathor-6
         layer_range: [0, 32]
-      - model: OmnicromsBrain/Eros_Scribe-7b
+      - model: MrRobotoAI/Hathor-2
         layer_range: [0, 32]
 merge_method: slerp
-base_model: OmnicromsBrain/Eros_Scribe-7b
+base_model: MrRobotoAI/Hathor-2
 parameters:
   t:
       - filter: self_attn

config.json

@@ -1,9 +1,13 @@
 {
-  "_name_or_path": "OmnicromsBrain/Eros_Scribe-7b",
+  "_name_or_path": "MrRobotoAI/Hathor-2",
   "architectures": [
     "MistralForCausalLM"
   ],
   "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "NousResearch/Yarn-Mistral-7b-128k--configuration_mistral.MistralConfig",
+    "AutoModelForCausalLM": "NousResearch/Yarn-Mistral-7b-128k--modeling_mistral_yarn.MistralForCausalLM"
+  },
   "bos_token_id": 1,
   "eos_token_id": 2,
   "hidden_act": "silu",
@@ -11,16 +15,23 @@
   "initializer_range": 0.02,
   "intermediate_size": 14336,
   "max_position_embeddings": 32768,
+  "max_sequence_length": 131072,
   "model_type": "mistral",
   "num_attention_heads": 32,
   "num_hidden_layers": 32,
   "num_key_value_heads": 8,
   "rms_norm_eps": 1e-05,
-  "rope_theta": 1000000.0,
-  "sliding_window": null,
+  "rope_scaling": {
+    "factor": 16.0,
+    "finetuned": true,
+    "original_max_position_embeddings": 8192,
+    "type": "yarn"
+  },
+  "rope_theta": 10000.0,
+  "sliding_window": 131072,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.41.0",
-  "use_cache": false,
+  "use_cache": true,
   "vocab_size": 32000
 }

mergekit_config.yml

@@ -1,11 +1,11 @@
 slices:
   - sources:
-      - model: MrRobotoAI/Nuit-4
+      - model: MrRobotoAI/Hathor-6
         layer_range: [0, 32]
-      - model: OmnicromsBrain/Eros_Scribe-7b
+      - model: MrRobotoAI/Hathor-2
         layer_range: [0, 32]
 merge_method: slerp
-base_model: OmnicromsBrain/Eros_Scribe-7b
+base_model: MrRobotoAI/Hathor-2
 parameters:
   t:
       - filter: self_attn

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:be6b122385d826510a00afbf747854ef1eb3771a384bb3c50ffb8c524196027f
+oid sha256:b22c79b24a3d6cbecc4d2462108657a7c507b582b1457818ce3e32379eb7e1c7
 size 4886547008

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cfa0bf73c472e374bdea0e03d38b057bd33e646e28c812a599bbb51eadb73e29
+oid sha256:6a454b27b6191e7b93751504d0f1ef47ec4aa6536990a12dcd642328cc80b565
 size 4915916176

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3af70ea087f148ba86101109a1cc836cfb4f1bf3fa7ebff0c62a3c1104d6b564
+oid sha256:ef587d0fd623b4db586526d413fa337bcc31226caea2efc0d087d06711d3bd2c
 size 4681034848

special_tokens_map.json

@@ -1,4 +1,9 @@
 {
+  "additional_special_tokens": [
+    "<unk>",
+    "<s>",
+    "</s>"
+  ],
   "bos_token": {
     "content": "<s>",
     "lstrip": false,
@@ -13,13 +18,6 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": {
-    "content": "<unk>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
   "unk_token": {
     "content": "<unk>",
     "lstrip": false,

tokenizer_config.json

@@ -1,7 +1,6 @@
 {
   "add_bos_token": true,
   "add_eos_token": false,
-  "add_prefix_space": true,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
@@ -28,17 +27,19 @@
       "special": true
     }
   },
+  "additional_special_tokens": [
+    "<unk>",
+    "<s>",
+    "</s>"
+  ],
   "bos_token": "<s>",
-  "chat_template": "{%- for idx in range(0, messages|length) -%}\n{%- if messages[idx]['role'] == 'user' -%}\n{%- if idx > 1 -%}\n{{- bos_token + '[INST] ' + messages[idx]['content'] + ' [/INST]' -}}\n{%- else -%}\n{{- messages[idx]['content'] + ' [/INST]' -}}\n{%- endif -%}\n{% elif messages[idx]['role'] == 'system' %}\n{{- '[INST] <<SYS>>\\n' + messages[idx]['content'] + '\\n<</SYS>>\\n\\n' -}}\n{%- elif messages[idx]['role'] == 'assistant' -%}\n{{- ' '  + messages[idx]['content'] + ' ' + eos_token -}}\n{% endif %}\n{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
-  "legacy": true,
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "<unk>",
-  "padding_side": "right",
+  "pad_token": null,
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
-  "use_default_system_prompt": false
+  "use_default_system_prompt": true
 }