keenanpepper commited on
Commit
34c0f1b
·
1 Parent(s): cfaf26e

Randomly initialized model with my hyperparams

Browse files
config.json CHANGED
@@ -1,19 +1,19 @@
1
  {
2
- "_name_or_path": "EleutherAI/gpt-neo-125M",
3
  "activation_function": "gelu_new",
4
  "architectures": [
5
- "GPTNeoForCausalLM"
6
  ],
7
  "attention_dropout": 0,
8
  "attention_layers": [
9
  "global",
10
- "local",
11
  "global",
12
- "local",
13
  "global",
14
- "local",
15
  "global",
16
- "local"
 
 
 
17
  ],
18
  "attention_types": [
19
  [
@@ -25,6 +25,7 @@
25
  ]
26
  ],
27
  "bos_token_id": 50256,
 
28
  "embed_dropout": 0,
29
  "eos_token_id": 50256,
30
  "gradient_checkpointing": false,
@@ -32,18 +33,18 @@
32
  "initializer_range": 0.02,
33
  "intermediate_size": null,
34
  "layer_norm_epsilon": 1e-05,
35
- "max_position_embeddings": 2048,
36
  "model_type": "gpt_neo",
37
  "num_heads": 16,
38
  "num_layers": 8,
39
  "resid_dropout": 0,
40
  "summary_activation": null,
41
- "summary_first_dropout": 0.1,
42
  "summary_proj_to_labels": true,
43
  "summary_type": "cls_index",
44
  "summary_use_proj": true,
45
  "torch_dtype": "float32",
46
- "transformers_version": "4.28.0",
47
  "use_cache": true,
48
  "vocab_size": 50257,
49
  "window_size": 256
 
1
  {
2
+ "_name_or_path": "roneneldan/TinyStories-3M",
3
  "activation_function": "gelu_new",
4
  "architectures": [
5
+ "GPTNeoModel"
6
  ],
7
  "attention_dropout": 0,
8
  "attention_layers": [
9
  "global",
 
10
  "global",
 
11
  "global",
 
12
  "global",
13
+ "global",
14
+ "global",
15
+ "global",
16
+ "global"
17
  ],
18
  "attention_types": [
19
  [
 
25
  ]
26
  ],
27
  "bos_token_id": 50256,
28
+ "classifier_dropout": 0.0,
29
  "embed_dropout": 0,
30
  "eos_token_id": 50256,
31
  "gradient_checkpointing": false,
 
33
  "initializer_range": 0.02,
34
  "intermediate_size": null,
35
  "layer_norm_epsilon": 1e-05,
36
+ "max_position_embeddings": 256,
37
  "model_type": "gpt_neo",
38
  "num_heads": 16,
39
  "num_layers": 8,
40
  "resid_dropout": 0,
41
  "summary_activation": null,
42
+ "summary_first_dropout": 0.0,
43
  "summary_proj_to_labels": true,
44
  "summary_type": "cls_index",
45
  "summary_use_proj": true,
46
  "torch_dtype": "float32",
47
+ "transformers_version": "4.45.2",
48
  "use_cache": true,
49
  "vocab_size": 50257,
50
  "window_size": 256
pytorch_model.bin → model.safetensors RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:0e4f93a86407a3f685520e48481a236c7b5e3a4f8e4fc8d1e58f3a748d97a37c
3
- size 66708541
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9601b9f78f60eb681e3e3a684b0c9252cb2536c7f7e16c48e39514bd3a3fdf6
3
+ size 32205816