Add files

.gitattributes

@@ -30,3 +30,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+logs/logs/xp3capmixnewcodelonglossseq/main_log.txt filter=lfs diff=lfs merge=lfs -text

config.json

@@ -0,0 +1,30 @@
+{
+  "apply_residual_connection_post_layernorm": false,
+  "attention_dropout": 0.0,
+  "attention_softmax_in_fp32": true,
+  "bias_dropout_fusion": true,
+  "architectures": [
+    "BloomModel"
+  ],
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 3,
+  "unk_token_id": 0,
+  "hidden_dropout": 0.0,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "masked_softmax_fusion": true,
+  "model_type": "bloom",
+  "n_embed": 2560,
+  "n_inner": null,
+  "n_layer": 30,
+  "num_attention_heads": 32,
+  "offset_alibi": 100,
+  "pretraining_tp": 4,
+  "seq_length": 2048,
+  "skip_bias_add": true,
+  "skip_bias_add_qkv": false,
+  "transformers_version": "4.20.0",
+  "use_cache": true,
+  "vocab_size": 250880
+}

evaluation_l1/Muennighoff_xstory_cloze/ar/Answer_Given_options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xstory_cloze",
+  "dataset_config_name": "ar",
+  "template_name": "Answer Given options",
+  "evaluation": {
+    "accuracy": 0.7511581733951026
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='ar', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Answer Given options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xstory_cloze/ar/Novel_Correct_Ending/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xstory_cloze",
+  "dataset_config_name": "ar",
+  "template_name": "Novel Correct Ending",
+  "evaluation": {
+    "accuracy": 0.7822634017207147
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='ar', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Novel Correct Ending', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xstory_cloze/ar/Story_Continuation_and_Options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xstory_cloze",
+  "dataset_config_name": "ar",
+  "template_name": "Story Continuation and Options",
+  "evaluation": {
+    "accuracy": 0.7908669755129054
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='ar', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Story Continuation and Options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xstory_cloze/eu/Answer_Given_options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xstory_cloze",
+  "dataset_config_name": "eu",
+  "template_name": "Answer Given options",
+  "evaluation": {
+    "accuracy": 0.6029119788219722
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='eu', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Answer Given options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xstory_cloze/eu/Choose_Story_Ending/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xstory_cloze",
+  "dataset_config_name": "eu",
+  "template_name": "Choose Story Ending",
+  "evaluation": {
+    "accuracy": 0.7094639311714097
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='eu', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Choose Story Ending', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xstory_cloze/eu/Generate_Ending/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xstory_cloze",
+  "dataset_config_name": "eu",
+  "template_name": "Generate Ending",
+  "evaluation": {
+    "accuracy": 0.5314361350099271
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='eu', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Generate Ending', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xstory_cloze/eu/Novel_Correct_Ending/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xstory_cloze",
+  "dataset_config_name": "eu",
+  "template_name": "Novel Correct Ending",
+  "evaluation": {
+    "accuracy": 0.6743878226340172
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='eu', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Novel Correct Ending', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xstory_cloze/hi/Answer_Given_options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xstory_cloze",
+  "dataset_config_name": "hi",
+  "template_name": "Answer Given options",
+  "evaluation": {
+    "accuracy": 0.6836532097948379
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='hi', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Answer Given options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xstory_cloze/hi/Choose_Story_Ending/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xstory_cloze",
+  "dataset_config_name": "hi",
+  "template_name": "Choose Story Ending",
+  "evaluation": {
+    "accuracy": 0.7888815354070152
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='hi', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Choose Story Ending', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xstory_cloze/hi/Generate_Ending/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xstory_cloze",
+  "dataset_config_name": "hi",
+  "template_name": "Generate Ending",
+  "evaluation": {
+    "accuracy": 0.5810721376571807
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='hi', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Generate Ending', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xstory_cloze/hi/Novel_Correct_Ending/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xstory_cloze",
+  "dataset_config_name": "hi",
+  "template_name": "Novel Correct Ending",
+  "evaluation": {
+    "accuracy": 0.7597617471872932
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='hi', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Novel Correct Ending', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xstory_cloze/hi/Story_Continuation_and_Options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xstory_cloze",
+  "dataset_config_name": "hi",
+  "template_name": "Story Continuation and Options",
+  "evaluation": {
+    "accuracy": 0.7723362011912641
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='hi', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Story Continuation and Options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xstory_cloze/id/Answer_Given_options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xstory_cloze",
+  "dataset_config_name": "id",
+  "template_name": "Answer Given options",
+  "evaluation": {
+    "accuracy": 0.7127729980145598
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='id', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Answer Given options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xstory_cloze/id/Choose_Story_Ending/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xstory_cloze",
+  "dataset_config_name": "id",
+  "template_name": "Choose Story Ending",
+  "evaluation": {
+    "accuracy": 0.8299139642620781
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='id', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Choose Story Ending', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xstory_cloze/id/Generate_Ending/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xstory_cloze",
+  "dataset_config_name": "id",
+  "template_name": "Generate Ending",
+  "evaluation": {
+    "accuracy": 0.6035737921906023
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='id', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Generate Ending', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xstory_cloze/id/Novel_Correct_Ending/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xstory_cloze",
+  "dataset_config_name": "id",
+  "template_name": "Novel Correct Ending",
+  "evaluation": {
+    "accuracy": 0.7180675049636003
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='id', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Novel Correct Ending', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xstory_cloze/id/Story_Continuation_and_Options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xstory_cloze",
+  "dataset_config_name": "id",
+  "template_name": "Story Continuation and Options",
+  "evaluation": {
+    "accuracy": 0.8021178027796162
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='id', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Story Continuation and Options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xstory_cloze/zh/Answer_Given_options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xstory_cloze",
+  "dataset_config_name": "zh",
+  "template_name": "Answer Given options",
+  "evaluation": {
+    "accuracy": 0.786234281932495
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Answer Given options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xstory_cloze/zh/Choose_Story_Ending/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xstory_cloze",
+  "dataset_config_name": "zh",
+  "template_name": "Choose Story Ending",
+  "evaluation": {
+    "accuracy": 0.8431502316346791
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Choose Story Ending', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xstory_cloze/zh/Generate_Ending/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xstory_cloze",
+  "dataset_config_name": "zh",
+  "template_name": "Generate Ending",
+  "evaluation": {
+    "accuracy": 0.6022501654533422
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Generate Ending', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xstory_cloze/zh/Novel_Correct_Ending/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xstory_cloze",
+  "dataset_config_name": "zh",
+  "template_name": "Novel Correct Ending",
+  "evaluation": {
+    "accuracy": 0.8365320979483786
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Novel Correct Ending', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xstory_cloze/zh/Story_Continuation_and_Options/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xstory_cloze",
+  "dataset_config_name": "zh",
+  "template_name": "Story Continuation and Options",
+  "evaluation": {
+    "accuracy": 0.8259430840502978
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xstory_cloze', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='validation', target_max_length=256, template_config_name='en', template_name='Story Continuation and Options', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xwinograd/en/Replace/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xwinograd",
+  "dataset_config_name": "en",
+  "template_name": "Replace",
+  "evaluation": {
+    "accuracy": 0.5922580645161291
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='Replace', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xwinograd/en/True_or_False/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xwinograd",
+  "dataset_config_name": "en",
+  "template_name": "True or False",
+  "evaluation": {
+    "accuracy": 0.5333333333333333
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='True or False', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xwinograd/en/does_underscore_refer_to/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xwinograd",
+  "dataset_config_name": "en",
+  "template_name": "does underscore refer to",
+  "evaluation": {
+    "accuracy": 0.5483870967741935
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='does underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xwinograd/en/stand_for/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xwinograd",
+  "dataset_config_name": "en",
+  "template_name": "stand for",
+  "evaluation": {
+    "accuracy": 0.5006451612903225
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='stand for', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xwinograd/en/underscore_refer_to/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xwinograd",
+  "dataset_config_name": "en",
+  "template_name": "underscore refer to",
+  "evaluation": {
+    "accuracy": 0.5574193548387096
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='en', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xwinograd/fr/Replace/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xwinograd",
+  "dataset_config_name": "fr",
+  "template_name": "Replace",
+  "evaluation": {
+    "accuracy": 0.4939759036144578
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='Replace', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xwinograd/fr/True_or_False/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xwinograd",
+  "dataset_config_name": "fr",
+  "template_name": "True or False",
+  "evaluation": {
+    "accuracy": 0.5060240963855421
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='True or False', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xwinograd/fr/does_underscore_refer_to/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xwinograd",
+  "dataset_config_name": "fr",
+  "template_name": "does underscore refer to",
+  "evaluation": {
+    "accuracy": 0.4819277108433735
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='does underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xwinograd/fr/stand_for/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xwinograd",
+  "dataset_config_name": "fr",
+  "template_name": "stand for",
+  "evaluation": {
+    "accuracy": 0.4939759036144578
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='fr', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='stand for', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xwinograd/pt/Replace/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xwinograd",
+  "dataset_config_name": "pt",
+  "template_name": "Replace",
+  "evaluation": {
+    "accuracy": 0.5285171102661597
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='pt', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='Replace', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xwinograd/pt/True_or_False/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xwinograd",
+  "dataset_config_name": "pt",
+  "template_name": "True or False",
+  "evaluation": {
+    "accuracy": 0.5361216730038023
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='pt', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='True or False', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xwinograd/pt/does_underscore_refer_to/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xwinograd",
+  "dataset_config_name": "pt",
+  "template_name": "does underscore refer to",
+  "evaluation": {
+    "accuracy": 0.5247148288973384
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='pt', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='does underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xwinograd/pt/stand_for/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xwinograd",
+  "dataset_config_name": "pt",
+  "template_name": "stand for",
+  "evaluation": {
+    "accuracy": 0.4828897338403042
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='pt', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='stand for', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xwinograd/pt/underscore_refer_to/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xwinograd",
+  "dataset_config_name": "pt",
+  "template_name": "underscore refer to",
+  "evaluation": {
+    "accuracy": 0.5019011406844106
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='pt', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xwinograd/zh/Replace/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xwinograd",
+  "dataset_config_name": "zh",
+  "template_name": "Replace",
+  "evaluation": {
+    "accuracy": 0.6091269841269841
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='Replace', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/Muennighoff_xwinograd/zh/underscore_refer_to/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "Muennighoff/xwinograd",
+  "dataset_config_name": "zh",
+  "template_name": "underscore refer to",
+  "evaluation": {
+    "accuracy": 0.5674603174603174
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='zh', dataset_name='Muennighoff/xwinograd', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='test', target_max_length=256, template_config_name='en', template_name='underscore refer to', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/anli/dev_r1/GPT-3_style/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "anli",
+  "dataset_config_name": "dev_r1",
+  "template_name": "GPT-3 style",
+  "evaluation": {
+    "accuracy": 0.401
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='dev_r1', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r1', target_max_length=256, template_config_name=None, template_name='GPT-3 style', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/anli/dev_r1/MNLI_crowdsource/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "anli",
+  "dataset_config_name": "dev_r1",
+  "template_name": "MNLI crowdsource",
+  "evaluation": {
+    "accuracy": 0.399
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='dev_r1', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r1', target_max_length=256, template_config_name=None, template_name='MNLI crowdsource', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/anli/dev_r1/can_we_infer/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "anli",
+  "dataset_config_name": "dev_r1",
+  "template_name": "can we infer",
+  "evaluation": {
+    "accuracy": 0.386
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='dev_r1', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r1', target_max_length=256, template_config_name=None, template_name='can we infer', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/anli/dev_r1/guaranteed_possible_impossible/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "anli",
+  "dataset_config_name": "dev_r1",
+  "template_name": "guaranteed/possible/impossible",
+  "evaluation": {
+    "accuracy": 0.33
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='dev_r1', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r1', target_max_length=256, template_config_name=None, template_name='guaranteed/possible/impossible', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/anli/dev_r1/justified_in_saying/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "anli",
+  "dataset_config_name": "dev_r1",
+  "template_name": "justified in saying",
+  "evaluation": {
+    "accuracy": 0.374
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='dev_r1', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r1', target_max_length=256, template_config_name=None, template_name='justified in saying', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/anli/dev_r2/GPT-3_style/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "anli",
+  "dataset_config_name": "dev_r2",
+  "template_name": "GPT-3 style",
+  "evaluation": {
+    "accuracy": 0.355
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='dev_r2', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r2', target_max_length=256, template_config_name=None, template_name='GPT-3 style', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/anli/dev_r2/MNLI_crowdsource/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "anli",
+  "dataset_config_name": "dev_r2",
+  "template_name": "MNLI crowdsource",
+  "evaluation": {
+    "accuracy": 0.368
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='dev_r2', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r2', target_max_length=256, template_config_name=None, template_name='MNLI crowdsource', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/anli/dev_r2/can_we_infer/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "anli",
+  "dataset_config_name": "dev_r2",
+  "template_name": "can we infer",
+  "evaluation": {
+    "accuracy": 0.335
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='dev_r2', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r2', target_max_length=256, template_config_name=None, template_name='can we infer', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/anli/dev_r2/guaranteed_possible_impossible/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "anli",
+  "dataset_config_name": "dev_r2",
+  "template_name": "guaranteed/possible/impossible",
+  "evaluation": {
+    "accuracy": 0.331
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='dev_r2', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r2', target_max_length=256, template_config_name=None, template_name='guaranteed/possible/impossible', tokenizer_name=None, use_slow_tokenizer=False)"
+}

evaluation_l1/anli/dev_r2/justified_in_saying/results.json

@@ -0,0 +1,9 @@
+{
+  "dataset_name": "anli",
+  "dataset_config_name": "dev_r2",
+  "template_name": "justified in saying",
+  "evaluation": {
+    "accuracy": 0.346
+  },
+  "arguments": "Namespace(config_name=None, dataset_config_name='dev_r2', dataset_name='anli', debug=False, dtype='float16', max_length=2048, model_name_or_path='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b', output_dir='/gpfsscratch/rech/six/commun/experiments/muennighoff/bloomckpt/2b5t0/bloomz-3b/evaluation', pad_to_max_length=False, per_device_eval_batch_size=8, prefixlm=False, split='dev_r2', target_max_length=256, template_config_name=None, template_name='justified in saying', tokenizer_name=None, use_slow_tokenizer=False)"
+}