Upload folder using huggingface_hub

README.md

@@ -1,7 +1,8 @@
 ---
 base_model:
-- MrRobotoAI/MrRoboto-BASE-v2.1-8b-64k
-- WeMake/Llama-3-8B-Instruct-V41-1048k
+- MrRobotoAI/MrRoboto-BASE-v2-8b-64k
+- MrRobotoAI/MrRoboto-BASE-v2-Unholy-8b-64k
+- MrRobotoAI/Llama-3-8B-Uncensored-0.2
 library_name: transformers
 tags:
 - mergekit
@@ -15,49 +16,27 @@ This is a merge of pre-trained language models created using [mergekit](https://
 ## Merge Details
 ### Merge Method
 
-This model was merged using the [Linear DARE](https://arxiv.org/abs/2311.03099) merge method using [WeMake/Llama-3-8B-Instruct-V41-1048k](https://huggingface.co/WeMake/Llama-3-8B-Instruct-V41-1048k) as a base.
+This model was merged using the [Model Stock](https://arxiv.org/abs/2403.19522) merge method using [MrRobotoAI/MrRoboto-BASE-v2-Unholy-8b-64k](https://huggingface.co/MrRobotoAI/MrRoboto-BASE-v2-Unholy-8b-64k) as a base.
 
 ### Models Merged
 
 The following models were included in the merge:
-* [MrRobotoAI/MrRoboto-BASE-v2.1-8b-64k](https://huggingface.co/MrRobotoAI/MrRoboto-BASE-v2.1-8b-64k)
+* [MrRobotoAI/MrRoboto-BASE-v2-8b-64k](https://huggingface.co/MrRobotoAI/MrRoboto-BASE-v2-8b-64k)
+* [MrRobotoAI/Llama-3-8B-Uncensored-0.2](https://huggingface.co/MrRobotoAI/Llama-3-8B-Uncensored-0.2)
 
 ### Configuration
 
 The following YAML configuration was used to produce this model:
 
 ```yaml
-merge_method: dare_linear
 models:
-  - model: WeMake/Llama-3-8B-Instruct-V41-1048k
-    parameters:
-      weight:
-        - filter: v_proj
-          value: [0.8, 0.8, 0.55, 0.45, 0.35, 0.25, 0.35, 0.45, 0.55, 0.8, 0.8]
-        - filter: o_proj
-          value: [0.8, 0.8, 0.55, 0.45, 0.35, 0.25, 0.35, 0.45, 0.55, 0.8, 0.8]
-        - filter: up_proj
-          value: [0.8, 0.8, 0.55, 0.45, 0.35, 0.25, 0.35, 0.45, 0.55, 0.8, 0.8]
-        - filter: gate_proj
-          value: [0.8, 0.8, 0.55, 0.45, 0.35, 0.25, 0.35, 0.45, 0.55, 0.8, 0.8]
-        - filter: down_proj
-          value: [0.8, 0.8, 0.55, 0.45, 0.35, 0.25, 0.35, 0.45, 0.55, 0.8, 0.8]
-        - value: 1
-  - model: MrRobotoAI/MrRoboto-BASE-v2.1-8b-64k
-    parameters:
-      weight:
-        - filter: v_proj
-          value: [0.2, 0.2, 0.45, 0.55, 0.65, 0.75, 0.65, 0.55, 0.45, 0.2, 0.2]
-        - filter: o_proj
-          value: [0.2, 0.2, 0.45, 0.55, 0.65, 0.75, 0.65, 0.55, 0.45, 0.2, 0.2]
-        - filter: up_proj
-          value: [0.2, 0.2, 0.45, 0.55, 0.65, 0.75, 0.65, 0.55, 0.45, 0.2, 0.2]
-        - filter: gate_proj
-          value: [0.2, 0.2, 0.45, 0.55, 0.65, 0.75, 0.65, 0.55, 0.45, 0.2, 0.2]
-        - filter: down_proj
-          value: [0.2, 0.2, 0.45, 0.55, 0.65, 0.75, 0.65, 0.55, 0.45, 0.2, 0.2]
-        - value: 0
-base_model: WeMake/Llama-3-8B-Instruct-V41-1048k
-tokenizer_source: base
-dtype: bfloat16
+
+  - model: MrRobotoAI/Llama-3-8B-Uncensored-0.2
+  - model: MrRobotoAI/MrRoboto-BASE-v2-8b-64k
+  - model: MrRobotoAI/MrRoboto-BASE-v2-Unholy-8b-64k
+  
+merge_method: model_stock
+base_model: MrRobotoAI/MrRoboto-BASE-v2-Unholy-8b-64k
+normalize: true
+dtype: float16
 ```

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "WeMake/Llama-3-8B-Instruct-V41-1048k",
+  "_name_or_path": "MrRobotoAI/MrRoboto-BASE-v2-Unholy-8b-64k",
   "architectures": [
     "LlamaForCausalLM"
   ],
@@ -23,7 +23,7 @@
   "rope_scaling": null,
   "rope_theta": 2804339835.0,
   "tie_word_embeddings": false,
-  "torch_dtype": "bfloat16",
+  "torch_dtype": "float16",
   "transformers_version": "4.48.2",
   "use_cache": true,
   "vocab_size": 128256

mergekit_config.yml

@@ -1,33 +1,10 @@
-merge_method: dare_linear
 models:
-  - model: WeMake/Llama-3-8B-Instruct-V41-1048k
-    parameters:
-      weight:
-        - filter: v_proj
-          value: [0.8, 0.8, 0.55, 0.45, 0.35, 0.25, 0.35, 0.45, 0.55, 0.8, 0.8]
-        - filter: o_proj
-          value: [0.8, 0.8, 0.55, 0.45, 0.35, 0.25, 0.35, 0.45, 0.55, 0.8, 0.8]
-        - filter: up_proj
-          value: [0.8, 0.8, 0.55, 0.45, 0.35, 0.25, 0.35, 0.45, 0.55, 0.8, 0.8]
-        - filter: gate_proj
-          value: [0.8, 0.8, 0.55, 0.45, 0.35, 0.25, 0.35, 0.45, 0.55, 0.8, 0.8]
-        - filter: down_proj
-          value: [0.8, 0.8, 0.55, 0.45, 0.35, 0.25, 0.35, 0.45, 0.55, 0.8, 0.8]
-        - value: 1
-  - model: MrRobotoAI/MrRoboto-BASE-v2.1-8b-64k
-    parameters:
-      weight:
-        - filter: v_proj
-          value: [0.2, 0.2, 0.45, 0.55, 0.65, 0.75, 0.65, 0.55, 0.45, 0.2, 0.2]
-        - filter: o_proj
-          value: [0.2, 0.2, 0.45, 0.55, 0.65, 0.75, 0.65, 0.55, 0.45, 0.2, 0.2]
-        - filter: up_proj
-          value: [0.2, 0.2, 0.45, 0.55, 0.65, 0.75, 0.65, 0.55, 0.45, 0.2, 0.2]
-        - filter: gate_proj
-          value: [0.2, 0.2, 0.45, 0.55, 0.65, 0.75, 0.65, 0.55, 0.45, 0.2, 0.2]
-        - filter: down_proj
-          value: [0.2, 0.2, 0.45, 0.55, 0.65, 0.75, 0.65, 0.55, 0.45, 0.2, 0.2]
-        - value: 0
-base_model: WeMake/Llama-3-8B-Instruct-V41-1048k
-tokenizer_source: base
-dtype: bfloat16
+
+  - model: MrRobotoAI/Llama-3-8B-Uncensored-0.2
+  - model: MrRobotoAI/MrRoboto-BASE-v2-8b-64k
+  - model: MrRobotoAI/MrRoboto-BASE-v2-Unholy-8b-64k
+  
+merge_method: model_stock
+base_model: MrRobotoAI/MrRoboto-BASE-v2-Unholy-8b-64k
+normalize: true
+dtype: float16

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:35b3732002e2dab64cb02382dafcad89f558b29fbcafd7edd31c12e2ecb49d3e
-size 4953586384
+oid sha256:39eea101144d0037488633a3b9ef931855e36aeaed6ed921182ebb9f01270390
+size 4953586328

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8e7c3d8a583855fdcb93195bca3b2da144794c3197e88c1e5daa838156e90e82
-size 4999819336
+oid sha256:3c009b3ea7a3419c2a406770acf098ccc6c31a2809b63ca0bc0f0743bade7c29
+size 4999819232

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:10d0f9b94244059dc5dbdcd003832935d2c6ba0f8856b186ab157f0e68eaf44d
-size 4915916144
+oid sha256:4e42f2be4c50451d98cb409c91a4700842aeae4af7ca655c30faf2386e0b0561
+size 4915916048

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f2305713ec18dd14c2182baec5f56e68cdea9eab160cd58ed8551837a98dbb15
-size 1191234472
+oid sha256:b3e3378c375c8987bb51e22b1fdf6472f56a91f322ea54231fcaa6a7b9f4a883
+size 1191234448