Isotonic commited on
Commit
be37f92
1 Parent(s): e8e14a0

Upload folder using huggingface_hub

Browse files
README.md ADDED
@@ -0,0 +1,177 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ tags:
4
+ - moe
5
+ - merge
6
+ - mergekit
7
+ - lazymergekit
8
+ - Felladrin/Minueza-32M-Base
9
+ - Felladrin/Minueza-32M-UltraChat
10
+ ---
11
+
12
+ # Mixnueza-6x32M-MoE
13
+
14
+ Mixnueza-6x32M-MoE is a Mixure of Experts (MoE) made with the following models using [LazyMergekit](https://colab.research.google.com/drive/1obulZ1ROXHjYLn6PPZJwRR6GzgQogxxb?usp=sharing):
15
+ * [Felladrin/Minueza-32M-Base](https://huggingface.co/Felladrin/Minueza-32M-Base)
16
+ * [Felladrin/Minueza-32M-UltraChat](https://huggingface.co/Felladrin/Minueza-32M-UltraChat)
17
+ * [Felladrin/Minueza-32M-Base](https://huggingface.co/Felladrin/Minueza-32M-Base)
18
+ * [Felladrin/Minueza-32M-UltraChat](https://huggingface.co/Felladrin/Minueza-32M-UltraChat)
19
+ * [Felladrin/Minueza-32M-Base](https://huggingface.co/Felladrin/Minueza-32M-Base)
20
+ * [Felladrin/Minueza-32M-UltraChat](https://huggingface.co/Felladrin/Minueza-32M-UltraChat)
21
+
22
+ ## 🧩 Configuration
23
+
24
+ ```yamlbase_model: Felladrin/Minueza-32M-UltraChat
25
+ experts:
26
+ - source_model: Felladrin/Minueza-32M-Base
27
+ positive_prompts:
28
+ - "reasoning"
29
+ - "logic"
30
+ - "problem-solving"
31
+ - "critical thinking"
32
+ - "analysis"
33
+ - "synthesis"
34
+ - "evaluation"
35
+ - "decision-making"
36
+ - "judgment"
37
+ - "insight"
38
+ negative_prompts:
39
+ - "programming"
40
+ - "storytelling"
41
+ - "legal"
42
+ - "finance"
43
+
44
+ - source_model: Felladrin/Minueza-32M-UltraChat
45
+ positive_prompts:
46
+ - "program"
47
+ - "software"
48
+ - "develop"
49
+ - "build"
50
+ - "create"
51
+ - "design"
52
+ - "implement"
53
+ - "debug"
54
+ - "test"
55
+ - "code"
56
+ - "python"
57
+ - "programming"
58
+ - "function"
59
+ negative_prompts:
60
+ - "reasoning"
61
+ - "storytelling"
62
+ - "legal"
63
+ - "finance"
64
+
65
+ - source_model: Felladrin/Minueza-32M-Base
66
+ positive_prompts:
67
+ - "storytelling"
68
+ - "narrative"
69
+ - "fiction"
70
+ - "creative writing"
71
+ - "plot"
72
+ - "characters"
73
+ - "dialogue"
74
+ - "setting"
75
+ - "emotion"
76
+ - "imagination"
77
+ - "scene"
78
+ - "story"
79
+ - "character"
80
+ negative_prompts:
81
+ - "reasoning"
82
+ - "programming"
83
+ - "legal"
84
+ - "finance"
85
+
86
+ - source_model: Felladrin/Minueza-32M-UltraChat
87
+ positive_prompts:
88
+ - "chat"
89
+ - "conversation"
90
+ - "dialogue"
91
+ - "discuss"
92
+ - "share thoughts"
93
+ - "explore ideas"
94
+ - "personal assistant"
95
+ - "friendly helper"
96
+ negative_prompts:
97
+ - "reasoning"
98
+ - "programming"
99
+ - "storytelling"
100
+ - "legal"
101
+ - "finance"
102
+
103
+ - source_model: Felladrin/Minueza-32M-Base
104
+ positive_prompts:
105
+ - "law"
106
+ - "legal"
107
+ - "attorney"
108
+ - "lawyer"
109
+ - "court"
110
+ - "contract"
111
+ - "criminal"
112
+ - "evidence"
113
+ - "procedure"
114
+ - "contracts"
115
+ - "mergers & acquisitions"
116
+ - "corporate governance"
117
+ - "intellectual property"
118
+ - "employment law"
119
+ - "international trade"
120
+ - "competition law"
121
+ - "antitrust"
122
+ - "litigation"
123
+ - "arbitration"
124
+ - "mediation"
125
+ negative_prompts:
126
+ - "reasoning"
127
+ - "programming"
128
+ - "storytelling"
129
+ - "finance"
130
+
131
+ - source_model: Felladrin/Minueza-32M-UltraChat
132
+ positive_prompts:
133
+ - "personal finance"
134
+ - "budgeting"
135
+ - "investing"
136
+ - "retirement planning"
137
+ - "debt management"
138
+ - "financial education"
139
+ - "financial"
140
+ - "money"
141
+ - "investment"
142
+ - "banking"
143
+ - "stock"
144
+ - "bond"
145
+ - "portfolio"
146
+ - "risk"
147
+ - "return"
148
+ negative_prompts:
149
+ - "reasoning"
150
+ - "programming"
151
+ - "storytelling"
152
+ - "legal"
153
+ ```
154
+
155
+ ## 💻 Usage
156
+
157
+ ```python
158
+ !pip install -qU transformers bitsandbytes accelerate
159
+
160
+ from transformers import AutoTokenizer
161
+ import transformers
162
+ import torch
163
+
164
+ model = "Isotonic/Mixnueza-6x32M-MoE"
165
+
166
+ tokenizer = AutoTokenizer.from_pretrained(model)
167
+ pipeline = transformers.pipeline(
168
+ "text-generation",
169
+ model=model,
170
+ model_kwargs={"torch_dtype": torch.float16, "load_in_4bit": True},
171
+ )
172
+
173
+ messages = [{"role": "user", "content": "Explain what a Mixture of Experts is in less than 100 words."}]
174
+ prompt = pipeline.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
175
+ outputs = pipeline(prompt, max_new_tokens=256, do_sample=True, temperature=0.7, top_k=50, top_p=0.95)
176
+ print(outputs[0]["generated_text"])
177
+ ```
added_tokens.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "<|im_end|>": 32000,
3
+ "<|im_start|>": 32001
4
+ }
config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "Felladrin/Minueza-32M-UltraChat",
3
+ "architectures": [
4
+ "MixtralForCausalLM"
5
+ ],
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 1,
8
+ "eos_token_id": 32000,
9
+ "hidden_act": "silu",
10
+ "hidden_size": 312,
11
+ "initializer_range": 0.02,
12
+ "intermediate_size": 1092,
13
+ "max_position_embeddings": 2048,
14
+ "model_type": "mixtral",
15
+ "num_attention_heads": 12,
16
+ "num_experts_per_tok": 2,
17
+ "num_hidden_layers": 10,
18
+ "num_key_value_heads": 4,
19
+ "num_local_experts": 6,
20
+ "output_router_logits": false,
21
+ "rms_norm_eps": 1e-06,
22
+ "rope_theta": 10000.0,
23
+ "router_aux_loss_coef": 0.001,
24
+ "sliding_window": null,
25
+ "tie_word_embeddings": false,
26
+ "torch_dtype": "float32",
27
+ "transformers_version": "4.39.3",
28
+ "use_cache": true,
29
+ "vocab_size": 32002
30
+ }
mergekit_moe_config.yml ADDED
@@ -0,0 +1,129 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ base_model: Felladrin/Minueza-32M-UltraChat
2
+ experts:
3
+ - source_model: Felladrin/Minueza-32M-Base
4
+ positive_prompts:
5
+ - "reasoning"
6
+ - "logic"
7
+ - "problem-solving"
8
+ - "critical thinking"
9
+ - "analysis"
10
+ - "synthesis"
11
+ - "evaluation"
12
+ - "decision-making"
13
+ - "judgment"
14
+ - "insight"
15
+ negative_prompts:
16
+ - "programming"
17
+ - "storytelling"
18
+ - "legal"
19
+ - "finance"
20
+
21
+ - source_model: Felladrin/Minueza-32M-UltraChat
22
+ positive_prompts:
23
+ - "program"
24
+ - "software"
25
+ - "develop"
26
+ - "build"
27
+ - "create"
28
+ - "design"
29
+ - "implement"
30
+ - "debug"
31
+ - "test"
32
+ - "code"
33
+ - "python"
34
+ - "programming"
35
+ - "function"
36
+ negative_prompts:
37
+ - "reasoning"
38
+ - "storytelling"
39
+ - "legal"
40
+ - "finance"
41
+
42
+ - source_model: Felladrin/Minueza-32M-Base
43
+ positive_prompts:
44
+ - "storytelling"
45
+ - "narrative"
46
+ - "fiction"
47
+ - "creative writing"
48
+ - "plot"
49
+ - "characters"
50
+ - "dialogue"
51
+ - "setting"
52
+ - "emotion"
53
+ - "imagination"
54
+ - "scene"
55
+ - "story"
56
+ - "character"
57
+ negative_prompts:
58
+ - "reasoning"
59
+ - "programming"
60
+ - "legal"
61
+ - "finance"
62
+
63
+ - source_model: Felladrin/Minueza-32M-UltraChat
64
+ positive_prompts:
65
+ - "chat"
66
+ - "conversation"
67
+ - "dialogue"
68
+ - "discuss"
69
+ - "share thoughts"
70
+ - "explore ideas"
71
+ - "personal assistant"
72
+ - "friendly helper"
73
+ negative_prompts:
74
+ - "reasoning"
75
+ - "programming"
76
+ - "storytelling"
77
+ - "legal"
78
+ - "finance"
79
+
80
+ - source_model: Felladrin/Minueza-32M-Base
81
+ positive_prompts:
82
+ - "law"
83
+ - "legal"
84
+ - "attorney"
85
+ - "lawyer"
86
+ - "court"
87
+ - "contract"
88
+ - "criminal"
89
+ - "evidence"
90
+ - "procedure"
91
+ - "contracts"
92
+ - "mergers & acquisitions"
93
+ - "corporate governance"
94
+ - "intellectual property"
95
+ - "employment law"
96
+ - "international trade"
97
+ - "competition law"
98
+ - "antitrust"
99
+ - "litigation"
100
+ - "arbitration"
101
+ - "mediation"
102
+ negative_prompts:
103
+ - "reasoning"
104
+ - "programming"
105
+ - "storytelling"
106
+ - "finance"
107
+
108
+ - source_model: Felladrin/Minueza-32M-UltraChat
109
+ positive_prompts:
110
+ - "personal finance"
111
+ - "budgeting"
112
+ - "investing"
113
+ - "retirement planning"
114
+ - "debt management"
115
+ - "financial education"
116
+ - "financial"
117
+ - "money"
118
+ - "investment"
119
+ - "banking"
120
+ - "stock"
121
+ - "bond"
122
+ - "portfolio"
123
+ - "risk"
124
+ - "return"
125
+ negative_prompts:
126
+ - "reasoning"
127
+ - "programming"
128
+ - "storytelling"
129
+ - "legal"
model-00001-of-00001.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:258ce1a6e5dad27d97ac4e9d00aeadab8debe7f99cdb8ac372ceebd1357329c0
3
+ size 335699144
model.safetensors.index.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"metadata": {"mergekit_version": "0.0.4"}, "weight_map": {"model.embed_tokens.weight": "model-00001-of-00001.safetensors", "model.norm.weight": "model-00001-of-00001.safetensors", "lm_head.weight": "model-00001-of-00001.safetensors", "model.layers.0.input_layernorm.weight": "model-00001-of-00001.safetensors", "model.layers.1.input_layernorm.weight": "model-00001-of-00001.safetensors", "model.layers.2.input_layernorm.weight": "model-00001-of-00001.safetensors", "model.layers.3.input_layernorm.weight": "model-00001-of-00001.safetensors", "model.layers.4.input_layernorm.weight": "model-00001-of-00001.safetensors", "model.layers.5.input_layernorm.weight": "model-00001-of-00001.safetensors", "model.layers.6.input_layernorm.weight": "model-00001-of-00001.safetensors", "model.layers.7.input_layernorm.weight": "model-00001-of-00001.safetensors", "model.layers.8.input_layernorm.weight": "model-00001-of-00001.safetensors", "model.layers.9.input_layernorm.weight": "model-00001-of-00001.safetensors", "model.layers.0.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00001.safetensors", "model.layers.0.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00001.safetensors", "model.layers.0.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00001.safetensors", "model.layers.0.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00001.safetensors", "model.layers.0.block_sparse_moe.experts.4.w3.weight": "model-00001-of-00001.safetensors", "model.layers.0.block_sparse_moe.experts.5.w3.weight": "model-00001-of-00001.safetensors", "model.layers.1.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00001.safetensors", "model.layers.1.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00001.safetensors", "model.layers.1.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00001.safetensors", "model.layers.1.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00001.safetensors", "model.layers.1.block_sparse_moe.experts.4.w3.weight": "model-00001-of-00001.safetensors", "model.layers.1.block_sparse_moe.experts.5.w3.weight": "model-00001-of-00001.safetensors", "model.layers.2.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00001.safetensors", "model.layers.2.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00001.safetensors", "model.layers.2.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00001.safetensors", "model.layers.2.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00001.safetensors", "model.layers.2.block_sparse_moe.experts.4.w3.weight": "model-00001-of-00001.safetensors", "model.layers.2.block_sparse_moe.experts.5.w3.weight": "model-00001-of-00001.safetensors", "model.layers.3.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00001.safetensors", "model.layers.3.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00001.safetensors", "model.layers.3.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00001.safetensors", "model.layers.3.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00001.safetensors", "model.layers.3.block_sparse_moe.experts.4.w3.weight": "model-00001-of-00001.safetensors", "model.layers.3.block_sparse_moe.experts.5.w3.weight": "model-00001-of-00001.safetensors", "model.layers.4.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00001.safetensors", "model.layers.4.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00001.safetensors", "model.layers.4.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00001.safetensors", "model.layers.4.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00001.safetensors", "model.layers.4.block_sparse_moe.experts.4.w3.weight": "model-00001-of-00001.safetensors", "model.layers.4.block_sparse_moe.experts.5.w3.weight": "model-00001-of-00001.safetensors", "model.layers.5.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00001.safetensors", "model.layers.5.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00001.safetensors", "model.layers.5.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00001.safetensors", "model.layers.5.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00001.safetensors", "model.layers.5.block_sparse_moe.experts.4.w3.weight": "model-00001-of-00001.safetensors", "model.layers.5.block_sparse_moe.experts.5.w3.weight": "model-00001-of-00001.safetensors", "model.layers.6.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00001.safetensors", "model.layers.6.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00001.safetensors", "model.layers.6.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00001.safetensors", "model.layers.6.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00001.safetensors", "model.layers.6.block_sparse_moe.experts.4.w3.weight": "model-00001-of-00001.safetensors", "model.layers.6.block_sparse_moe.experts.5.w3.weight": "model-00001-of-00001.safetensors", "model.layers.7.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00001.safetensors", "model.layers.7.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00001.safetensors", "model.layers.7.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00001.safetensors", "model.layers.7.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00001.safetensors", "model.layers.7.block_sparse_moe.experts.4.w3.weight": "model-00001-of-00001.safetensors", "model.layers.7.block_sparse_moe.experts.5.w3.weight": "model-00001-of-00001.safetensors", "model.layers.8.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00001.safetensors", "model.layers.8.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00001.safetensors", "model.layers.8.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00001.safetensors", "model.layers.8.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00001.safetensors", "model.layers.8.block_sparse_moe.experts.4.w3.weight": "model-00001-of-00001.safetensors", "model.layers.8.block_sparse_moe.experts.5.w3.weight": "model-00001-of-00001.safetensors", "model.layers.9.block_sparse_moe.experts.0.w3.weight": "model-00001-of-00001.safetensors", "model.layers.9.block_sparse_moe.experts.1.w3.weight": "model-00001-of-00001.safetensors", "model.layers.9.block_sparse_moe.experts.2.w3.weight": "model-00001-of-00001.safetensors", "model.layers.9.block_sparse_moe.experts.3.w3.weight": "model-00001-of-00001.safetensors", "model.layers.9.block_sparse_moe.experts.4.w3.weight": "model-00001-of-00001.safetensors", "model.layers.9.block_sparse_moe.experts.5.w3.weight": "model-00001-of-00001.safetensors", "model.layers.0.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00001.safetensors", "model.layers.0.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00001.safetensors", "model.layers.0.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00001.safetensors", "model.layers.0.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00001.safetensors", "model.layers.0.block_sparse_moe.experts.4.w2.weight": "model-00001-of-00001.safetensors", "model.layers.0.block_sparse_moe.experts.5.w2.weight": "model-00001-of-00001.safetensors", "model.layers.1.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00001.safetensors", "model.layers.1.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00001.safetensors", "model.layers.1.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00001.safetensors", "model.layers.1.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00001.safetensors", "model.layers.1.block_sparse_moe.experts.4.w2.weight": "model-00001-of-00001.safetensors", "model.layers.1.block_sparse_moe.experts.5.w2.weight": "model-00001-of-00001.safetensors", "model.layers.2.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00001.safetensors", "model.layers.2.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00001.safetensors", "model.layers.2.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00001.safetensors", "model.layers.2.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00001.safetensors", "model.layers.2.block_sparse_moe.experts.4.w2.weight": "model-00001-of-00001.safetensors", "model.layers.2.block_sparse_moe.experts.5.w2.weight": "model-00001-of-00001.safetensors", "model.layers.3.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00001.safetensors", "model.layers.3.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00001.safetensors", "model.layers.3.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00001.safetensors", "model.layers.3.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00001.safetensors", "model.layers.3.block_sparse_moe.experts.4.w2.weight": "model-00001-of-00001.safetensors", "model.layers.3.block_sparse_moe.experts.5.w2.weight": "model-00001-of-00001.safetensors", "model.layers.4.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00001.safetensors", "model.layers.4.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00001.safetensors", "model.layers.4.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00001.safetensors", "model.layers.4.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00001.safetensors", "model.layers.4.block_sparse_moe.experts.4.w2.weight": "model-00001-of-00001.safetensors", "model.layers.4.block_sparse_moe.experts.5.w2.weight": "model-00001-of-00001.safetensors", "model.layers.5.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00001.safetensors", "model.layers.5.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00001.safetensors", "model.layers.5.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00001.safetensors", "model.layers.5.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00001.safetensors", "model.layers.5.block_sparse_moe.experts.4.w2.weight": "model-00001-of-00001.safetensors", "model.layers.5.block_sparse_moe.experts.5.w2.weight": "model-00001-of-00001.safetensors", "model.layers.6.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00001.safetensors", "model.layers.6.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00001.safetensors", "model.layers.6.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00001.safetensors", "model.layers.6.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00001.safetensors", "model.layers.6.block_sparse_moe.experts.4.w2.weight": "model-00001-of-00001.safetensors", "model.layers.6.block_sparse_moe.experts.5.w2.weight": "model-00001-of-00001.safetensors", "model.layers.7.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00001.safetensors", "model.layers.7.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00001.safetensors", "model.layers.7.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00001.safetensors", "model.layers.7.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00001.safetensors", "model.layers.7.block_sparse_moe.experts.4.w2.weight": "model-00001-of-00001.safetensors", "model.layers.7.block_sparse_moe.experts.5.w2.weight": "model-00001-of-00001.safetensors", "model.layers.8.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00001.safetensors", "model.layers.8.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00001.safetensors", "model.layers.8.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00001.safetensors", "model.layers.8.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00001.safetensors", "model.layers.8.block_sparse_moe.experts.4.w2.weight": "model-00001-of-00001.safetensors", "model.layers.8.block_sparse_moe.experts.5.w2.weight": "model-00001-of-00001.safetensors", "model.layers.9.block_sparse_moe.experts.0.w2.weight": "model-00001-of-00001.safetensors", "model.layers.9.block_sparse_moe.experts.1.w2.weight": "model-00001-of-00001.safetensors", "model.layers.9.block_sparse_moe.experts.2.w2.weight": "model-00001-of-00001.safetensors", "model.layers.9.block_sparse_moe.experts.3.w2.weight": "model-00001-of-00001.safetensors", "model.layers.9.block_sparse_moe.experts.4.w2.weight": "model-00001-of-00001.safetensors", "model.layers.9.block_sparse_moe.experts.5.w2.weight": "model-00001-of-00001.safetensors", "model.layers.0.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00001.safetensors", "model.layers.0.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00001.safetensors", "model.layers.0.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00001.safetensors", "model.layers.0.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00001.safetensors", "model.layers.0.block_sparse_moe.experts.4.w1.weight": "model-00001-of-00001.safetensors", "model.layers.0.block_sparse_moe.experts.5.w1.weight": "model-00001-of-00001.safetensors", "model.layers.1.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00001.safetensors", "model.layers.1.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00001.safetensors", "model.layers.1.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00001.safetensors", "model.layers.1.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00001.safetensors", "model.layers.1.block_sparse_moe.experts.4.w1.weight": "model-00001-of-00001.safetensors", "model.layers.1.block_sparse_moe.experts.5.w1.weight": "model-00001-of-00001.safetensors", "model.layers.2.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00001.safetensors", "model.layers.2.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00001.safetensors", "model.layers.2.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00001.safetensors", "model.layers.2.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00001.safetensors", "model.layers.2.block_sparse_moe.experts.4.w1.weight": "model-00001-of-00001.safetensors", "model.layers.2.block_sparse_moe.experts.5.w1.weight": "model-00001-of-00001.safetensors", "model.layers.3.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00001.safetensors", "model.layers.3.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00001.safetensors", "model.layers.3.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00001.safetensors", "model.layers.3.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00001.safetensors", "model.layers.3.block_sparse_moe.experts.4.w1.weight": "model-00001-of-00001.safetensors", "model.layers.3.block_sparse_moe.experts.5.w1.weight": "model-00001-of-00001.safetensors", "model.layers.4.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00001.safetensors", "model.layers.4.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00001.safetensors", "model.layers.4.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00001.safetensors", "model.layers.4.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00001.safetensors", "model.layers.4.block_sparse_moe.experts.4.w1.weight": "model-00001-of-00001.safetensors", "model.layers.4.block_sparse_moe.experts.5.w1.weight": "model-00001-of-00001.safetensors", "model.layers.5.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00001.safetensors", "model.layers.5.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00001.safetensors", "model.layers.5.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00001.safetensors", "model.layers.5.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00001.safetensors", "model.layers.5.block_sparse_moe.experts.4.w1.weight": "model-00001-of-00001.safetensors", "model.layers.5.block_sparse_moe.experts.5.w1.weight": "model-00001-of-00001.safetensors", "model.layers.6.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00001.safetensors", "model.layers.6.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00001.safetensors", "model.layers.6.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00001.safetensors", "model.layers.6.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00001.safetensors", "model.layers.6.block_sparse_moe.experts.4.w1.weight": "model-00001-of-00001.safetensors", "model.layers.6.block_sparse_moe.experts.5.w1.weight": "model-00001-of-00001.safetensors", "model.layers.7.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00001.safetensors", "model.layers.7.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00001.safetensors", "model.layers.7.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00001.safetensors", "model.layers.7.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00001.safetensors", "model.layers.7.block_sparse_moe.experts.4.w1.weight": "model-00001-of-00001.safetensors", "model.layers.7.block_sparse_moe.experts.5.w1.weight": "model-00001-of-00001.safetensors", "model.layers.8.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00001.safetensors", "model.layers.8.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00001.safetensors", "model.layers.8.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00001.safetensors", "model.layers.8.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00001.safetensors", "model.layers.8.block_sparse_moe.experts.4.w1.weight": "model-00001-of-00001.safetensors", "model.layers.8.block_sparse_moe.experts.5.w1.weight": "model-00001-of-00001.safetensors", "model.layers.9.block_sparse_moe.experts.0.w1.weight": "model-00001-of-00001.safetensors", "model.layers.9.block_sparse_moe.experts.1.w1.weight": "model-00001-of-00001.safetensors", "model.layers.9.block_sparse_moe.experts.2.w1.weight": "model-00001-of-00001.safetensors", "model.layers.9.block_sparse_moe.experts.3.w1.weight": "model-00001-of-00001.safetensors", "model.layers.9.block_sparse_moe.experts.4.w1.weight": "model-00001-of-00001.safetensors", "model.layers.9.block_sparse_moe.experts.5.w1.weight": "model-00001-of-00001.safetensors", "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00001.safetensors", "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00001.safetensors", "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00001.safetensors", "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00001.safetensors", "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00001.safetensors", "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00001.safetensors", "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00001.safetensors", "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00001.safetensors", "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00001.safetensors", "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00001.safetensors", "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00001.safetensors", "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00001.safetensors", "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00001.safetensors", "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00001.safetensors", "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00001.safetensors", "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00001.safetensors", "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00001.safetensors", "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00001.safetensors", "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00001.safetensors", "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00001.safetensors", "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00001.safetensors", "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00001.safetensors", "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00001.safetensors", "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00001.safetensors", "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00001.safetensors", "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00001.safetensors", "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00001.safetensors", "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00001.safetensors", "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00001.safetensors", "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00001.safetensors", "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00001.safetensors", "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00001.safetensors", "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00001.safetensors", "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00001.safetensors", "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00001.safetensors", "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00001.safetensors", "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00001.safetensors", "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00001.safetensors", "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00001.safetensors", "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00001.safetensors", "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00001.safetensors", "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00001.safetensors", "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00001.safetensors", "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00001.safetensors", "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00001.safetensors", "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00001.safetensors", "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00001.safetensors", "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00001.safetensors", "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00001.safetensors", "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00001.safetensors", "model.layers.0.block_sparse_moe.gate.weight": "model-00001-of-00001.safetensors", "model.layers.1.block_sparse_moe.gate.weight": "model-00001-of-00001.safetensors", "model.layers.2.block_sparse_moe.gate.weight": "model-00001-of-00001.safetensors", "model.layers.3.block_sparse_moe.gate.weight": "model-00001-of-00001.safetensors", "model.layers.4.block_sparse_moe.gate.weight": "model-00001-of-00001.safetensors", "model.layers.5.block_sparse_moe.gate.weight": "model-00001-of-00001.safetensors", "model.layers.6.block_sparse_moe.gate.weight": "model-00001-of-00001.safetensors", "model.layers.7.block_sparse_moe.gate.weight": "model-00001-of-00001.safetensors", "model.layers.8.block_sparse_moe.gate.weight": "model-00001-of-00001.safetensors", "model.layers.9.block_sparse_moe.gate.weight": "model-00001-of-00001.safetensors"}}
special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "<|im_end|>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "<s>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
3
+ size 493443
tokenizer_config.json ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_eos_token": false,
4
+ "added_tokens_decoder": {
5
+ "0": {
6
+ "content": "<unk>",
7
+ "lstrip": false,
8
+ "normalized": false,
9
+ "rstrip": false,
10
+ "single_word": false,
11
+ "special": true
12
+ },
13
+ "1": {
14
+ "content": "<s>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false,
19
+ "special": true
20
+ },
21
+ "2": {
22
+ "content": "</s>",
23
+ "lstrip": false,
24
+ "normalized": false,
25
+ "rstrip": false,
26
+ "single_word": false,
27
+ "special": true
28
+ },
29
+ "32000": {
30
+ "content": "<|im_end|>",
31
+ "lstrip": false,
32
+ "normalized": false,
33
+ "rstrip": false,
34
+ "single_word": false,
35
+ "special": true
36
+ },
37
+ "32001": {
38
+ "content": "<|im_start|>",
39
+ "lstrip": false,
40
+ "normalized": false,
41
+ "rstrip": false,
42
+ "single_word": false,
43
+ "special": true
44
+ }
45
+ },
46
+ "additional_special_tokens": [],
47
+ "bos_token": "<s>",
48
+ "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
49
+ "clean_up_tokenization_spaces": false,
50
+ "eos_token": "<|im_end|>",
51
+ "legacy": true,
52
+ "model_max_length": 2048,
53
+ "pad_token": "<s>",
54
+ "sp_model_kwargs": {},
55
+ "spaces_between_special_tokens": false,
56
+ "tokenizer_class": "LlamaTokenizer",
57
+ "unk_token": "<unk>",
58
+ "use_default_system_prompt": false
59
+ }