Add files using upload-large-folder tool

adapter2_config.json

@@ -0,0 +1,29 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "./models/ColSmolVLM-256M-Base",
+  "bias": "none",
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": "gaussian",
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": "(.*(model.text_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)",
+  "task_type": "FEATURE_EXTRACTION",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_config.json

@@ -0,0 +1,29 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "./models/ColSmolVLM-256M-Base",
+  "bias": "none",
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": "gaussian",
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": "(.*(model.text_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)",
+  "task_type": "FEATURE_EXTRACTION",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7e32974ae0949c6ce82561c2a883cb41fef7e1dd67349dcbd23ae294e111821
+size 39135840

added_tokens.json

@@ -0,0 +1,42 @@
+{
+  "<end_of_utterance>": 49191,
+  "<fake_token_around_image>": 49189,
+  "<global-img>": 49152,
+  "<image>": 49190,
+  "<row_1_col_1>": 49153,
+  "<row_1_col_2>": 49154,
+  "<row_1_col_3>": 49155,
+  "<row_1_col_4>": 49156,
+  "<row_1_col_5>": 49157,
+  "<row_1_col_6>": 49158,
+  "<row_2_col_1>": 49159,
+  "<row_2_col_2>": 49160,
+  "<row_2_col_3>": 49161,
+  "<row_2_col_4>": 49162,
+  "<row_2_col_5>": 49163,
+  "<row_2_col_6>": 49164,
+  "<row_3_col_1>": 49165,
+  "<row_3_col_2>": 49166,
+  "<row_3_col_3>": 49167,
+  "<row_3_col_4>": 49168,
+  "<row_3_col_5>": 49169,
+  "<row_3_col_6>": 49170,
+  "<row_4_col_1>": 49171,
+  "<row_4_col_2>": 49172,
+  "<row_4_col_3>": 49173,
+  "<row_4_col_4>": 49174,
+  "<row_4_col_5>": 49175,
+  "<row_4_col_6>": 49176,
+  "<row_5_col_1>": 49177,
+  "<row_5_col_2>": 49178,
+  "<row_5_col_3>": 49179,
+  "<row_5_col_4>": 49180,
+  "<row_5_col_5>": 49181,
+  "<row_5_col_6>": 49182,
+  "<row_6_col_1>": 49183,
+  "<row_6_col_2>": 49184,
+  "<row_6_col_3>": 49185,
+  "<row_6_col_4>": 49186,
+  "<row_6_col_5>": 49187,
+  "<row_6_col_6>": 49188
+}

chat_template.json

@@ -0,0 +1,3 @@
+{
+  "chat_template": "<|im_start|>{% for message in messages %}{{message['role'].capitalize()}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '<image>' }}{% endif %}{% endfor %}<|endoftext|>\n{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}"
+}

checkpoint-11082/README.md

@@ -0,0 +1,202 @@
+---
+base_model: ./models/ColSmolVLM-256M-Base
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.14.0

checkpoint-11082/adapter_config.json

@@ -0,0 +1,29 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "./models/ColSmolVLM-256M-Base",
+  "bias": "none",
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": "gaussian",
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": "(.*(model.text_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)",
+  "task_type": "FEATURE_EXTRACTION",
+  "use_dora": false,
+  "use_rslora": false
+}

checkpoint-11082/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7e32974ae0949c6ce82561c2a883cb41fef7e1dd67349dcbd23ae294e111821
+size 39135840

checkpoint-11082/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7ee0f69c5cc1d6d145e92982afa5bd9ad18cb63202e662af394db1d03ea9336
+size 78504446

checkpoint-11082/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c653672b3faef730c291bb1cb7ca4e4761c3c7e2114dfe4db070e1dd92e7d15
+size 14244

checkpoint-11082/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6492f98570ecb136d3a3c030cd8d38f5d102dfae6ace1f3bfc893278748c61b
+size 1064

checkpoint-11082/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c0134daeadf97734a4e9fe5bb83517012ff54a24865a2e385a561c6333f763c
+size 5304

config.json

@@ -0,0 +1,265 @@
+{
+  "_name_or_path": "models/SmolVLM-256M-Base",
+  "architectures": [
+    "ColIdefics3"
+  ],
+  "image_token_id": 49190,
+  "model_type": "idefics3",
+  "scale_factor": 4,
+  "text_config": {
+    "_attn_implementation_autoset": false,
+    "_flash_attn_2_enabled": true,
+    "_name_or_path": "/fsx/m4/experiments/local_experiment_dir/tr_341_vsmollm2_05b/opt_step-27750/unwrapped_model",
+    "add_cross_attention": false,
+    "architectures": [
+      "VLlama3ForCausalLM"
+    ],
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 1,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 2,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "head_dim": 64,
+    "hidden_act": "silu",
+    "hidden_size": 576,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.041666666666666664,
+    "intermediate_size": 1536,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "is_llama_config": true,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 8192,
+    "min_length": 0,
+    "mlp_bias": false,
+    "model_type": "llama",
+    "neftune_noise_alpha": 0.0,
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 9,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 30,
+    "num_key_value_heads": 3,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": 2,
+    "perceiver_config": {
+      "_attn_implementation_autoset": false,
+      "_name_or_path": "",
+      "add_cross_attention": false,
+      "architectures": null,
+      "attention_dropout": 0.0,
+      "bad_words_ids": null,
+      "begin_suppress_tokens": null,
+      "bos_token_id": null,
+      "chunk_size_feed_forward": 0,
+      "cross_attention_hidden_size": null,
+      "decoder_start_token_id": null,
+      "diversity_penalty": 0.0,
+      "do_sample": false,
+      "early_stopping": false,
+      "encoder_no_repeat_ngram_size": 0,
+      "eos_token_id": null,
+      "exponential_decay_length_penalty": null,
+      "finetuning_task": null,
+      "forced_bos_token_id": null,
+      "forced_eos_token_id": null,
+      "hidden_act": "silu",
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "is_decoder": false,
+      "is_encoder_decoder": false,
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "length_penalty": 1.0,
+      "max_length": 20,
+      "min_length": 0,
+      "model_type": "vllama3",
+      "no_repeat_ngram_size": 0,
+      "num_beam_groups": 1,
+      "num_beams": 1,
+      "num_key_value_heads": 1,
+      "num_return_sequences": 1,
+      "output_attentions": false,
+      "output_hidden_states": false,
+      "output_scores": false,
+      "pad_token_id": null,
+      "prefix": null,
+      "problem_type": null,
+      "pruned_heads": {},
+      "qk_layer_norms_perceiver": false,
+      "remove_invalid_values": false,
+      "repetition_penalty": 1.0,
+      "resampler_depth": 6,
+      "resampler_head_dim": 96,
+      "resampler_n_heads": 16,
+      "resampler_n_latents": 64,
+      "return_dict": true,
+      "return_dict_in_generate": false,
+      "sep_token_id": null,
+      "suppress_tokens": null,
+      "task_specific_params": null,
+      "temperature": 1.0,
+      "tf_legacy_loss": false,
+      "tie_encoder_decoder": false,
+      "tie_word_embeddings": true,
+      "tokenizer_class": null,
+      "top_k": 50,
+      "top_p": 1.0,
+      "torch_dtype": null,
+      "torchscript": false,
+      "transformers_version": "4.46.0",
+      "typical_p": 1.0,
+      "use_bfloat16": false
+    },
+    "pixel_shuffle_factor": 4,
+    "prefix": null,
+    "pretraining_tp": 1,
+    "problem_type": null,
+    "pruned_heads": {},
+    "qk_layer_norms": false,
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rms_norm_eps": 1e-05,
+    "rope_interleaved": false,
+    "rope_scaling": null,
+    "rope_theta": 100000,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": false,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "transformers.js_config": {
+      "kv_cache_dtype": {
+        "fp16": "float16",
+        "q4f16": "float16"
+      }
+    },
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": true,
+    "use_resampler": false,
+    "vocab_size": 49192
+  },
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.3",
+  "use_cache": true,
+  "vision_config": {
+    "_attn_implementation_autoset": false,
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": null,
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 768,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "image_size": 512,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_norm_eps": 1e-06,
+    "length_penalty": 1.0,
+    "max_image_size": {
+      "longest_edge": 512
+    },
+    "max_length": 20,
+    "min_length": 0,
+    "model_type": "idefics3",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 12,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_channels": 3,
+    "num_hidden_layers": 12,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "patch_size": 16,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "size": {
+      "longest_edge": 2048
+    },
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": false,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": null,
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false
+  },
+  "vocab_size": 49192
+}

dump/tmp_adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "./models/ColSmolVLM-256M-Base",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": "gaussian",
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": "(.*(model.text_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)",
+  "task_type": "FEATURE_EXTRACTION",
+  "use_dora": false,
+  "use_rslora": false
+}

dump/tmp_adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d307b9b7fafeea5a9e5f42a92b6d952353c74972f9b0d09e837f4fde3b4f9064
+size 19598104

git_hash.txt

@@ -0,0 +1 @@
+8b4f75e476bddc6c34a02722761bd3ada6ac0d3d

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00cef852358625358be817768b2b45ba85f80e76279a01b75e23c2b6c1cd9450
+size 456304568

preprocessor_config.json

@@ -0,0 +1,28 @@
+{
+  "do_convert_rgb": true,
+  "do_image_splitting": true,
+  "do_normalize": true,
+  "do_pad": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "Idefics3ImageProcessor",
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "max_image_size": {
+    "longest_edge": 512
+  },
+  "processor_class": "ColIdefics3Processor",
+  "resample": 1,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "longest_edge": 2048
+  }
+}

processor_config.json

@@ -0,0 +1,4 @@
+{
+  "image_seq_len": 64,
+  "processor_class": "Idefics3Processor"
+}

results.json

@@ -0,0 +1 @@
+{"./data_dir/eval_vidore/arxivqa_test_subsampled": {"ndcg_at_1": 0.622, "ndcg_at_3": 0.67486, "ndcg_at_5": 0.69017, "ndcg_at_10": 0.70946, "ndcg_at_20": 0.72898, "ndcg_at_50": 0.73804, "ndcg_at_100": 0.74397, "map_at_1": 0.622, "map_at_3": 0.662, "map_at_5": 0.6703, "map_at_10": 0.67857, "map_at_20": 0.6841, "map_at_50": 0.68552, "map_at_100": 0.68607, "recall_at_1": 0.622, "recall_at_3": 0.712, "recall_at_5": 0.75, "recall_at_10": 0.808, "recall_at_20": 0.884, "recall_at_50": 0.93, "recall_at_100": 0.966, "precision_at_1": 0.622, "precision_at_3": 0.23733, "precision_at_5": 0.15, "precision_at_10": 0.0808, "precision_at_20": 0.0442, "precision_at_50": 0.0186, "precision_at_100": 0.00966, "mrr_at_1": 0.618, "mrr_at_3": 0.66, "mrr_at_5": 0.6692, "mrr_at_10": 0.6775777777777778, "mrr_at_20": 0.6824860893148819, "mrr_at_50": 0.6840505621420908, "mrr_at_100": 0.6845993176544162, "naucs_at_1_max": 0.46491355378995425, "naucs_at_1_std": -0.06263837723835318, "naucs_at_1_diff1": 0.897580000185275, "naucs_at_3_max": 0.5074962351081752, "naucs_at_3_std": -0.006120744926715115, "naucs_at_3_diff1": 0.8380872858484801, "naucs_at_5_max": 0.5076588173731037, "naucs_at_5_std": 0.00014233385661883033, "naucs_at_5_diff1": 0.818938775510204, "naucs_at_10_max": 0.5770523781854408, "naucs_at_10_std": 0.09178633684734683, "naucs_at_10_diff1": 0.7791151323981074, "naucs_at_20_max": 0.6430128856683821, "naucs_at_20_std": 0.16875963146332645, "naucs_at_20_diff1": 0.7221138397980255, "naucs_at_50_max": 0.6545151393890893, "naucs_at_50_std": 0.26363878884887104, "naucs_at_50_diff1": 0.7213151927437641, "naucs_at_100_max": 0.5877409787444364, "naucs_at_100_std": 0.19736914373592712, "naucs_at_100_diff1": 0.7009666611742652}, "./data_dir/eval_vidore/docvqa_test_subsampled": {"ndcg_at_1": 0.45676, "ndcg_at_3": 0.52515, "ndcg_at_5": 0.54233, "ndcg_at_10": 0.56915, "ndcg_at_20": 0.5816, "ndcg_at_50": 0.59505, "ndcg_at_100": 0.60258, "map_at_1": 0.45676, "map_at_3": 0.50887, "map_at_5": 0.51829, "map_at_10": 0.52954, "map_at_20": 0.53302, "map_at_50": 0.53529, "map_at_100": 0.53594, "recall_at_1": 0.45676, "recall_at_3": 0.57206, "recall_at_5": 0.61419, "recall_at_10": 0.69623, "recall_at_20": 0.74501, "recall_at_50": 0.81153, "recall_at_100": 0.85809, "precision_at_1": 0.45676, "precision_at_3": 0.19069, "precision_at_5": 0.12284, "precision_at_10": 0.06962, "precision_at_20": 0.03725, "precision_at_50": 0.01623, "precision_at_100": 0.00858, "mrr_at_1": 0.45454545454545453, "mrr_at_3": 0.5066518847006651, "mrr_at_5": 0.5170731707317073, "mrr_at_10": 0.5275815647766867, "mrr_at_20": 0.5311224770899007, "mrr_at_50": 0.5332465170438858, "mrr_at_100": 0.5339250823073732, "naucs_at_1_max": 0.3882948611231304, "naucs_at_1_std": 0.6051243703689616, "naucs_at_1_diff1": 0.8406058479800685, "naucs_at_3_max": 0.3234157964181825, "naucs_at_3_std": 0.6966233741677433, "naucs_at_3_diff1": 0.7729637232110695, "naucs_at_5_max": 0.27276717369737363, "naucs_at_5_std": 0.7006240836207518, "naucs_at_5_diff1": 0.747950791051886, "naucs_at_10_max": 0.16091474340315445, "naucs_at_10_std": 0.7256404433556597, "naucs_at_10_diff1": 0.6981512112733994, "naucs_at_20_max": 0.08633158860631607, "naucs_at_20_std": 0.7747045325493711, "naucs_at_20_diff1": 0.7160198478434746, "naucs_at_50_max": -0.04978574435251967, "naucs_at_50_std": 0.7706903740216889, "naucs_at_50_diff1": 0.6946330790179335, "naucs_at_100_max": -0.1803641306976603, "naucs_at_100_std": 0.8287710376301388, "naucs_at_100_diff1": 0.709766038885269}, "./data_dir/eval_vidore/syntheticDocQA_energy_test": {"ndcg_at_1": 0.87, "ndcg_at_3": 0.91786, "ndcg_at_5": 0.92603, "ndcg_at_10": 0.93226, "ndcg_at_20": 0.93226, "ndcg_at_50": 0.93422, "ndcg_at_100": 0.93422, "map_at_1": 0.87, "map_at_3": 0.90667, "map_at_5": 0.91117, "map_at_10": 0.9136, "map_at_20": 0.9136, "map_at_50": 0.9139, "map_at_100": 0.9139, "recall_at_1": 0.87, "recall_at_3": 0.95, "recall_at_5": 0.97, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.87, "precision_at_3": 0.31667, "precision_at_5": 0.194, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.88, "mrr_at_3": 0.9116666666666667, "mrr_at_5": 0.9181666666666666, "mrr_at_10": 0.9195952380952381, "mrr_at_20": 0.9195952380952381, "mrr_at_50": 0.9199077380952382, "mrr_at_100": 0.9199077380952382, "naucs_at_1_max": 0.3063763608087097, "naucs_at_1_std": -0.2671628526994001, "naucs_at_1_diff1": 0.9131674442716432, "naucs_at_3_max": 0.7082166199813253, "naucs_at_3_std": 0.1034547152194234, "naucs_at_3_diff1": 0.8627450980392143, "naucs_at_5_max": 0.8202614379084955, "naucs_at_5_std": 0.0704948646125114, "naucs_at_5_diff1": 0.8638344226579531, "naucs_at_10_max": 0.8692810457516413, "naucs_at_10_std": 0.5541549953314738, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 0.8692810457516413, "naucs_at_20_std": 0.5541549953314738, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "./data_dir/eval_vidore/tatdqa_test": {"ndcg_at_1": 0.58809, "ndcg_at_3": 0.70323, "ndcg_at_5": 0.73068, "ndcg_at_10": 0.75075, "ndcg_at_20": 0.75855, "ndcg_at_50": 0.76442, "ndcg_at_100": 0.7668, "map_at_1": 0.58809, "map_at_3": 0.67517, "map_at_5": 0.69036, "map_at_10": 0.69878, "map_at_20": 0.70099, "map_at_50": 0.70196, "map_at_100": 0.70218, "recall_at_1": 0.58809, "recall_at_3": 0.78433, "recall_at_5": 0.85115, "recall_at_10": 0.91252, "recall_at_20": 0.94289, "recall_at_50": 0.97205, "recall_at_100": 0.98663, "precision_at_1": 0.58809, "precision_at_3": 0.26144, "precision_at_5": 0.17023, "precision_at_10": 0.09125, "precision_at_20": 0.04714, "precision_at_50": 0.01944, "precision_at_100": 0.00987, "mrr_at_1": 0.5911300121506683, "mrr_at_3": 0.6764884568651276, "mrr_at_5": 0.6917071688942892, "mrr_at_10": 0.7006767247198595, "mrr_at_20": 0.702628923116812, "mrr_at_50": 0.7035729931078816, "mrr_at_100": 0.7038013299596423, "naucs_at_1_max": 0.16282263440251535, "naucs_at_1_std": -0.2675726048319812, "naucs_at_1_diff1": 0.7855612887331109, "naucs_at_3_max": 0.15869758594775502, "naucs_at_3_std": -0.25252459447099584, "naucs_at_3_diff1": 0.6394956014993703, "naucs_at_5_max": 0.19188935642904395, "naucs_at_5_std": -0.19455295056436547, "naucs_at_5_diff1": 0.6071937017845587, "naucs_at_10_max": 0.2647534611934194, "naucs_at_10_std": -0.054514027748884926, "naucs_at_10_diff1": 0.5405997520135849, "naucs_at_20_max": 0.3763911903387556, "naucs_at_20_std": 0.14592272943721332, "naucs_at_20_diff1": 0.4732245856774079, "naucs_at_50_max": 0.35454897123535783, "naucs_at_50_std": 0.275162782423997, "naucs_at_50_diff1": 0.4532394718684134, "naucs_at_100_max": 0.2978782618422668, "naucs_at_100_std": 0.2997014737148475, "naucs_at_100_diff1": 0.45555904105628975}, "./data_dir/eval_vidore/infovqa_test_subsampled": {"ndcg_at_1": 0.74494, "ndcg_at_3": 0.79976, "ndcg_at_5": 0.81213, "ndcg_at_10": 0.82846, "ndcg_at_20": 0.83297, "ndcg_at_50": 0.83835, "ndcg_at_100": 0.83933, "map_at_1": 0.74494, "map_at_3": 0.78711, "map_at_5": 0.79389, "map_at_10": 0.8006, "map_at_20": 0.80179, "map_at_50": 0.80272, "map_at_100": 0.8028, "recall_at_1": 0.74494, "recall_at_3": 0.83603, "recall_at_5": 0.8664, "recall_at_10": 0.917, "recall_at_20": 0.93522, "recall_at_50": 0.96154, "recall_at_100": 0.96761, "precision_at_1": 0.74494, "precision_at_3": 0.27868, "precision_at_5": 0.17328, "precision_at_10": 0.0917, "precision_at_20": 0.04676, "precision_at_50": 0.01923, "precision_at_100": 0.00968, "mrr_at_1": 0.742914979757085, "mrr_at_3": 0.7850877192982456, "mrr_at_5": 0.7931848852901484, "mrr_at_10": 0.7990416746995694, "mrr_at_20": 0.8004113500481683, "mrr_at_50": 0.801331531125384, "mrr_at_100": 0.8014399188597515, "naucs_at_1_max": 0.4965959718308746, "naucs_at_1_std": -0.0466933622092674, "naucs_at_1_diff1": 0.8685373039401096, "naucs_at_3_max": 0.5196845357871381, "naucs_at_3_std": 0.014152907602651955, "naucs_at_3_diff1": 0.8084743558786613, "naucs_at_5_max": 0.5925268631205242, "naucs_at_5_std": 0.11651891324846737, "naucs_at_5_diff1": 0.7815885765686577, "naucs_at_10_max": 0.581202610435799, "naucs_at_10_std": 0.23681516182108397, "naucs_at_10_diff1": 0.7349454780601268, "naucs_at_20_max": 0.7922518260743858, "naucs_at_20_std": 0.5495609276681086, "naucs_at_20_diff1": 0.7372706091805555, "naucs_at_50_max": 0.8510476089522382, "naucs_at_50_std": 0.7768090684036366, "naucs_at_50_diff1": 0.6938503074292259, "naucs_at_100_max": 0.8476068082679248, "naucs_at_100_std": 0.8111079242428052, "naucs_at_100_diff1": 0.7003229329465875}, "./data_dir/eval_vidore/syntheticDocQA_healthcare_industry_test": {"ndcg_at_1": 0.92, "ndcg_at_3": 0.95393, "ndcg_at_5": 0.9578, "ndcg_at_10": 0.96113, "ndcg_at_20": 0.96113, "ndcg_at_50": 0.96113, "ndcg_at_100": 0.96113, "map_at_1": 0.92, "map_at_3": 0.945, "map_at_5": 0.947, "map_at_10": 0.94843, "map_at_20": 0.94843, "map_at_50": 0.94843, "map_at_100": 0.94843, "recall_at_1": 0.92, "recall_at_3": 0.98, "recall_at_5": 0.99, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.92, "precision_at_3": 0.32667, "precision_at_5": 0.198, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.93, "mrr_at_3": 0.9516666666666667, "mrr_at_5": 0.9536666666666666, "mrr_at_10": 0.9550952380952381, "mrr_at_20": 0.9550952380952381, "mrr_at_50": 0.9550952380952381, "mrr_at_100": 0.9550952380952381, "naucs_at_1_max": 0.5140056022408969, "naucs_at_1_std": -0.16888422035480954, "naucs_at_1_diff1": 0.8211367880485536, "naucs_at_3_max": 1.0, "naucs_at_3_std": 0.3384687208216551, "naucs_at_3_diff1": 0.9346405228758099, "naucs_at_5_max": 1.0, "naucs_at_5_std": 0.12278244631185926, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "./data_dir/eval_vidore/tabfquad_test_subsampled": {"ndcg_at_1": 0.56429, "ndcg_at_3": 0.63444, "ndcg_at_5": 0.65026, "ndcg_at_10": 0.66294, "ndcg_at_20": 0.67546, "ndcg_at_50": 0.70544, "ndcg_at_100": 0.71155, "map_at_1": 0.56429, "map_at_3": 0.61786, "map_at_5": 0.62643, "map_at_10": 0.63165, "map_at_20": 0.63503, "map_at_50": 0.63965, "map_at_100": 0.64028, "recall_at_1": 0.56429, "recall_at_3": 0.68214, "recall_at_5": 0.72143, "recall_at_10": 0.76071, "recall_at_20": 0.81071, "recall_at_50": 0.96429, "recall_at_100": 1.0, "precision_at_1": 0.56429, "precision_at_3": 0.22738, "precision_at_5": 0.14429, "precision_at_10": 0.07607, "precision_at_20": 0.04054, "precision_at_50": 0.01929, "precision_at_100": 0.01, "mrr_at_1": 0.5642857142857143, "mrr_at_3": 0.6172619047619048, "mrr_at_5": 0.6267261904761904, "mrr_at_10": 0.6319331065759637, "mrr_at_20": 0.6352391827022797, "mrr_at_50": 0.6399410284324812, "mrr_at_100": 0.6405056533320372, "naucs_at_1_max": 0.250909164994346, "naucs_at_1_std": 0.10353784876791601, "naucs_at_1_diff1": 0.7091792557580451, "naucs_at_3_max": 0.19901092567688966, "naucs_at_3_std": 0.06257940172158412, "naucs_at_3_diff1": 0.6023933615512848, "naucs_at_5_max": 0.16905697042683365, "naucs_at_5_std": 0.025700327070190307, "naucs_at_5_diff1": 0.5975052344915358, "naucs_at_10_max": 0.11986463149253845, "naucs_at_10_std": 0.04068799417636617, "naucs_at_10_diff1": 0.5762841809353437, "naucs_at_20_max": 0.16545256557084664, "naucs_at_20_std": 0.0609616431669272, "naucs_at_20_diff1": 0.511002008899236, "naucs_at_50_max": 0.4613912231559318, "naucs_at_50_std": 0.2222222222222222, "naucs_at_50_diff1": 0.5568160597572381, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0}, "./data_dir/eval_vidore/syntheticDocQA_government_reports_test": {"ndcg_at_1": 0.79, "ndcg_at_3": 0.89464, "ndcg_at_5": 0.90238, "ndcg_at_10": 0.90883, "ndcg_at_20": 0.90883, "ndcg_at_50": 0.90883, "ndcg_at_100": 0.90883, "map_at_1": 0.79, "map_at_3": 0.87167, "map_at_5": 0.87567, "map_at_10": 0.87833, "map_at_20": 0.87833, "map_at_50": 0.87833, "map_at_100": 0.87833, "recall_at_1": 0.79, "recall_at_3": 0.96, "recall_at_5": 0.98, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.79, "precision_at_3": 0.32, "precision_at_5": 0.196, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.85, "mrr_at_3": 0.9016666666666667, "mrr_at_5": 0.9056666666666666, "mrr_at_10": 0.9085833333333333, "mrr_at_20": 0.9085833333333333, "mrr_at_50": 0.9085833333333333, "mrr_at_100": 0.9085833333333333, "naucs_at_1_max": 0.2326688628657996, "naucs_at_1_std": 0.03888234303989203, "naucs_at_1_diff1": 0.9258181643301988, "naucs_at_3_max": 0.7480158730158707, "naucs_at_3_std": 0.6259337068160535, "naucs_at_3_diff1": 0.8231792717086835, "naucs_at_5_max": 0.9346405228758136, "naucs_at_5_std": 0.540149393090577, "naucs_at_5_diff1": 0.9346405228758136, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "./data_dir/eval_vidore/shiftproject_test": {"ndcg_at_1": 0.48, "ndcg_at_3": 0.59202, "ndcg_at_5": 0.64626, "ndcg_at_10": 0.67623, "ndcg_at_20": 0.68119, "ndcg_at_50": 0.69074, "ndcg_at_100": 0.69411, "map_at_1": 0.48, "map_at_3": 0.565, "map_at_5": 0.5955, "map_at_10": 0.60836, "map_at_20": 0.60966, "map_at_50": 0.61104, "map_at_100": 0.61137, "recall_at_1": 0.48, "recall_at_3": 0.67, "recall_at_5": 0.8, "recall_at_10": 0.89, "recall_at_20": 0.91, "recall_at_50": 0.96, "recall_at_100": 0.98, "precision_at_1": 0.48, "precision_at_3": 0.22333, "precision_at_5": 0.16, "precision_at_10": 0.089, "precision_at_20": 0.0455, "precision_at_50": 0.0192, "precision_at_100": 0.0098, "mrr_at_1": 0.49, "mrr_at_3": 0.5883333333333334, "mrr_at_5": 0.6138333333333333, "mrr_at_10": 0.622702380952381, "mrr_at_20": 0.6240598470157294, "mrr_at_50": 0.6257612271725352, "mrr_at_100": 0.6259278938392019, "naucs_at_1_max": 0.10312797713559874, "naucs_at_1_std": -0.02315549910024338, "naucs_at_1_diff1": 0.6334947602413465, "naucs_at_3_max": -0.0512264901559134, "naucs_at_3_std": -0.15811481099016378, "naucs_at_3_diff1": 0.42873829798039664, "naucs_at_5_max": 0.20339660339660318, "naucs_at_5_std": 0.09702797202797177, "naucs_at_5_diff1": 0.4798701298701295, "naucs_at_10_max": 0.15213469633193094, "naucs_at_10_std": -0.08190877072416533, "naucs_at_10_diff1": 0.5727171205222922, "naucs_at_20_max": 0.265380226164541, "naucs_at_20_std": 0.09165888577653315, "naucs_at_20_diff1": 0.4984956945741262, "naucs_at_50_max": 0.2783613445378158, "naucs_at_50_std": 0.06430905695611724, "naucs_at_50_diff1": 0.1669000933706846, "naucs_at_100_max": -0.22035480859009582, "naucs_at_100_std": -0.22035480859009582, "naucs_at_100_diff1": -0.22035480859009582}, "./data_dir/eval_vidore/syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.92, "ndcg_at_3": 0.95655, "ndcg_at_5": 0.96085, "ndcg_at_10": 0.96085, "ndcg_at_20": 0.96085, "ndcg_at_50": 0.96085, "ndcg_at_100": 0.9625, "map_at_1": 0.92, "map_at_3": 0.94833, "map_at_5": 0.95083, "map_at_10": 0.95083, "map_at_20": 0.95083, "map_at_50": 0.95083, "map_at_100": 0.95098, "recall_at_1": 0.92, "recall_at_3": 0.98, "recall_at_5": 0.99, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_50": 0.99, "recall_at_100": 1.0, "precision_at_1": 0.92, "precision_at_3": 0.32667, "precision_at_5": 0.198, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_50": 0.0198, "precision_at_100": 0.01, "mrr_at_1": 0.92, "mrr_at_3": 0.9516666666666667, "mrr_at_5": 0.9516666666666667, "mrr_at_10": 0.9516666666666667, "mrr_at_20": 0.9516666666666667, "mrr_at_50": 0.9516666666666667, "mrr_at_100": 0.9518205128205127, "naucs_at_1_max": 0.5828081232492984, "naucs_at_1_std": 0.2532096171802052, "naucs_at_1_diff1": 0.8768674136321195, "naucs_at_3_max": 0.8611111111111119, "naucs_at_3_std": 0.0793650793650649, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 0.7222222222222276, "naucs_at_5_std": -0.5634920634920767, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 0.7222222222222276, "naucs_at_10_std": -0.5634920634920767, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 0.7222222222222276, "naucs_at_20_std": -0.5634920634920767, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": 0.7222222222222041, "naucs_at_50_std": -0.5634920634920583, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}}

special_tokens_map.json

@@ -0,0 +1,53 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<fake_token_around_image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<end_of_utterance>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,479 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49152": {
+      "content": "<global-img>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49153": {
+      "content": "<row_1_col_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49154": {
+      "content": "<row_1_col_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49155": {
+      "content": "<row_1_col_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49156": {
+      "content": "<row_1_col_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49157": {
+      "content": "<row_1_col_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49158": {
+      "content": "<row_1_col_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49159": {
+      "content": "<row_2_col_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49160": {
+      "content": "<row_2_col_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49161": {
+      "content": "<row_2_col_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49162": {
+      "content": "<row_2_col_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49163": {
+      "content": "<row_2_col_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49164": {
+      "content": "<row_2_col_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49165": {
+      "content": "<row_3_col_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49166": {
+      "content": "<row_3_col_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49167": {
+      "content": "<row_3_col_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49168": {
+      "content": "<row_3_col_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49169": {
+      "content": "<row_3_col_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49170": {
+      "content": "<row_3_col_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49171": {
+      "content": "<row_4_col_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49172": {
+      "content": "<row_4_col_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49173": {
+      "content": "<row_4_col_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49174": {
+      "content": "<row_4_col_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49175": {
+      "content": "<row_4_col_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49176": {
+      "content": "<row_4_col_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49177": {
+      "content": "<row_5_col_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49178": {
+      "content": "<row_5_col_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49179": {
+      "content": "<row_5_col_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49180": {
+      "content": "<row_5_col_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49181": {
+      "content": "<row_5_col_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49182": {
+      "content": "<row_5_col_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49183": {
+      "content": "<row_6_col_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49184": {
+      "content": "<row_6_col_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49185": {
+      "content": "<row_6_col_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49186": {
+      "content": "<row_6_col_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49187": {
+      "content": "<row_6_col_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49188": {
+      "content": "<row_6_col_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49189": {
+      "content": "<fake_token_around_image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49190": {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49191": {
+      "content": "<end_of_utterance>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<fake_token_around_image>",
+    "<image>",
+    "<end_of_utterance>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 8192,
+  "pad_token": "<|im_end|>",
+  "processor_class": "ColIdefics3Processor",
+  "tokenizer_class": "GPT2Tokenizer",
+  "truncation_side": "left",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

training_config.yml

@@ -0,0 +1,72 @@
+config:
+  (): colpali_engine.trainer.colmodel_training.ColModelTrainingConfig
+  output_dir: !path ../../../models/ColSmolVLM-256M-Base
+  processor:
+    (): colpali_engine.utils.transformers_wrappers.AllPurposeWrapper
+    class_to_instanciate: !ext colpali_engine.models.ColIdefics3Processor
+    pretrained_model_name_or_path:  "./models/ColSmolVLM-256M-Base"
+    # num_image_tokens: 2048
+    # max_length: 50
+
+  model:
+    (): colpali_engine.utils.transformers_wrappers.AllPurposeWrapper
+    class_to_instanciate: !ext colpali_engine.models.ColIdefics3
+    pretrained_model_name_or_path: "./models/ColSmolVLM-256M-Base"
+    torch_dtype:  !ext torch.bfloat16
+    # use_cache: false
+    attn_implementation: "flash_attention_2"
+#    device_map: "auto"
+#    quantization_config:
+#      (): transformers.BitsAndBytesConfig
+#      load_in_4bit: true
+#      bnb_4bit_quant_type: "nf4"
+#      bnb_4bit_compute_dtype:  "bfloat16"
+#      bnb_4bit_use_double_quant: true
+
+  dataset_loading_func: !ext colpali_engine.utils.dataset_transformation.load_train_set
+  eval_dataset_loader: !import ../data/test_data.yaml
+
+  # max_length: 50
+  run_eval: true
+  loss_func:
+    (): colpali_engine.loss.late_interaction_losses.ColbertPairwiseCELoss
+  tr_args:
+    (): transformers.training_args.TrainingArguments
+    output_dir: null
+    overwrite_output_dir: true
+    num_train_epochs: 3
+    per_device_train_batch_size: 8
+    gradient_checkpointing: true
+    gradient_checkpointing_kwargs: { "use_reentrant": false }
+    # gradient_checkpointing: true
+    # 6 x 8 gpus = 48 batch size
+    # gradient_accumulation_steps: 4
+    per_device_eval_batch_size: 8
+    eval_strategy: "steps"
+    dataloader_num_workers: 4
+    # bf16: true
+    save_steps: 500
+    logging_steps: 10
+    eval_steps: 100
+    warmup_steps: 100
+    learning_rate: 5e-4
+    save_total_limit: 1
+    resume_from_checkpoint: true
+    # optim: "paged_adamw_8bit"
+    # wandb logging
+    # wandb_project: "colqwen2"
+    # run_name: "colqwen2-ba32-nolora"
+    report_to: "wandb"
+
+
+  peft_config:
+    (): peft.LoraConfig
+    r: 32
+    lora_alpha: 32
+    lora_dropout: 0.1
+    init_lora_weights: "gaussian"
+    bias: "none"
+    task_type: "FEATURE_EXTRACTION"
+    target_modules: '(.*(model.text_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)'
+    # target_modules: '(.*(language_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)'
+