init space

TinyLLaVA_Factory/.gitignore

@@ -0,0 +1,63 @@
+# These are some examples of commonly ignored file patterns.
+# You should customize this list as applicable to your project.
+# Learn more about .gitignore:
+#     https://www.atlassian.com/git/tutorials/saving-changes/gitignore
+
+# Node artifact files
+node_modules/
+dist/
+
+# Compiled Java class files
+*.class
+
+# Compiled Python bytecode
+*.py[cod]
+
+# Log files
+*.log
+
+# Package files
+*.jar
+
+# Maven
+target/
+dist/
+
+# JetBrains IDE
+.idea/
+
+# Unit test reports
+TEST*.xml
+
+# Generated by MacOS
+.DS_Store
+
+# Generated by Windows
+Thumbs.db
+
+# Applications
+*.app
+*.exe
+*.war
+
+# Large media files
+*.mp4
+*.tiff
+*.avi
+*.flv
+*.mov
+*.wmv
+
+#
+.ipynb_checkpoints
+__pycache__
+*.egg-info
+.vscode/*
+.idea/*
+playground/
+wandb/*
+checkpoints/*
+.ipynb_checkpoints/*
+scripts/.ipynb_checkpoints/*
+test/*
+output/*

TinyLLaVA_Factory/CUSTOM_FINETUNE.md

@@ -0,0 +1,96 @@
+# Finetune TinyLLaVA with Custom Datasets
+
+Given the needs of finetuning with custom datasets, we provide a tutorial on how to custom finetune on our trained model, e.g. tinyllava/TinyLLaVA-Phi-2-SigLIP-3.1B (HF path).
+
+## Dataset Format
+
+Convert your data to a JSON file of a List of all samples. Sample metadata should contain `id` (a unique identifier), `image` (the path to the image), and `conversations` (the conversation data between human and AI).
+
+Here's an example of the [pokemon dataset](https://huggingface.co/datasets/lambdalabs/pokemon-blip-captions) turned into the data format:
+
+```json
+[
+  {
+        "id": "meiKqU2auAVK2vrtLhKGoJ",
+        "image": "pokemon/image/meiKqU2auAVK2vrtLhKGoJ.jpg",
+        "conversations": [
+            {
+                "from": "human",
+                "value": "<image>\nProvide a brief description of the given image."
+            },
+            {
+                "from": "gpt",
+                "value": "a drawing of a green pokemon with red eyes"
+            }
+        ]
+    }
+]
+```
+
+<details>
+You can use the following scripts to convert the Pokemon dataset to the above data format.
+<summary>converting data format</summary>
+  
+```python
+import shortuuid
+from datasets import load_dataset
+from PIL import Image
+import random
+import json
+import tqdm
+import os
+
+ds = load_dataset('lambdalabs/pokemon-blip-captions')
+pokemon_data = []
+
+pokemon_image_path = '/path/to/your/data/pokemon/image'
+pokemon_data_path = '/path/to/your/pokemon_blip_captions.json'
+
+description_list = [
+    "Describe the image concisely.",
+    "Provide a brief description of the given image.",
+    "Offer a succinct explanation of the picture presented.",
+    "Summarize the visual content of the image.",
+    "Give a short and clear explanation of the subsequent image.",
+    "Share a concise interpretation of the image provided.",
+    "Present a compact description of the photo's key features.",
+    "Relay a brief, clear account of the picture shown.",
+    "Render a clear and concise summary of the photo.",
+    "Write a terse but informative summary of the picture.",
+    "Create a compact narrative representing the image presented."
+]
+
+for sample in tqdm.tqdm(ds['train']):
+    uuid = shortuuid.uuid()
+    sample_dict = dict()
+    sample_dict['id'] = uuid
+    sample_dict['image'] = 'pokemon/image/' + uuid + '.jpg'
+    sample['image'].save(os.path.join(pokemon_image_path, uuid + '.jpg'))
+    conversations = [
+        {"from": "human", "value": "<image>\n" + random.choice(description_list)},
+        {"from": "gpt", "value": sample['text']}
+    ]
+    sample_dict['conversations'] = conversations
+    pokemon_data.append(sample_dict)
+
+with open(pokemon_data_path, 'w') as f:
+    json.dump(pokemon_data, f, indent=4)
+```
+
+</details>
+
+## Custom Finetune
+After acquiring the dataset following the above data format, you can finetune our trained model TinyLLaVA-Phi-2-SigLIP-3.1B checkpoint by using lora.
+
+- Replace data paths and `output_dir` with yours in `scripts/train/custom_finetune.sh`
+- Adjust your GPU ids (localhost) and `per_device_train_batch_size` in `scripts/train/custom_finetune.sh`.
+
+```bash
+bash scripts/train/custom_finetune.sh
+```
+
+## Evaluation with Custom Finetuned Model
+All of the models trained by TinyLLaVA Factory have the same evaluation procedure, no matter it is trained through custom finetune or through normal training. Please see the [Evaluation](https://tinyllava-factory.readthedocs.io/en/latest/Evaluation.html) section in our Doc.
+
+
+

TinyLLaVA_Factory/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [TinyLLaVA]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

TinyLLaVA_Factory/README.md

@@ -0,0 +1,428 @@
+<h2 align="center"> <a href="https://arxiv.org/abs/2402.14289">TinyLLaVA Factory</a><h5 align="center">
+
+[![hf_space](https://img.shields.io/badge/🤗-%20Open%20In%20HF-blue.svg)](https://huggingface.co/tinyllava) [![arXiv](https://img.shields.io/badge/Arxiv-2402.14289-b31b1b.svg?logo=arXiv)](https://arxiv.org/abs/2402.14289) [![arXiv](https://img.shields.io/badge/Arxiv-2405.11788-b31b1b.svg?logo=arXiv)](https://arxiv.org/abs/2405.11788)[![License](https://img.shields.io/badge/License-Apache%202.0-yellow)](https://github.com/TinyLLaVA/TinyLLaVA_Factory/blob/main/LICENSE) [![Doc](https://img.shields.io/badge/Doc-Document-logo=read%20the%20docs&logoColor=white&label=Doc)](https://tinyllava-factory.readthedocs.io/en/latest/) [![Demo](https://img.shields.io/badge/Demo-Demo-red.svg)](http://8843843nmph5.vicp.fun/#/)
+
+![architecture](./assets/architecture.jpg)
+
+## &#x1F389; News
+* **[2024.08.13]**  A simple [visualizaiton tool](https://github.com/TinyLLaVA/TinyLLaVA_Factory/tree/main/tinyllava_visualizer) for interpreting the prediction of TinyLLaVA is added.
+* **[2024.05.21]**  Our paper: [TinyLLaVA Factory: A Modularized Codebase for Small-scale Large Multimodal Models](https://arxiv.org/abs/2405.11788) is released!
+* **[2024.05.15]** [TinyLLaVA Factory](https://github.com/TinyLLaVA/TinyLLaVA_Factory), our new codebase, is released!  **Note that the old codebase, TinyLLaVABench, is moved to the [tinyllava_bench](https://github.com/TinyLLaVA/TinyLLaVA_Factory/tree/tinyllava_bench) branch.**
+* **[2024.05.04]**  [TinyLLaVA Demo](http://8843843nmph5.vicp.fun/#/) is released!
+* **[2024.02.21]**  Our paper: [TinyLLaVA: A Framework of Small-scale Large Multimodal Models](https://arxiv.org/abs/2402.14289) is released!
+
+## &#x1F525; Takeaways
+- Our best model, [TinyLLaVA-Phi-2-SigLIP-3.1B](https://huggingface.co/tinyllava/TinyLLaVA-Phi-2-SigLIP-3.1B), achieves better overall performance against existing 7B models such as LLaVA-1.5 and Qwen-VL.
+
+- TinyLLaVA Factory is an open-source modular codebase for small-scale large multimodal models (LMMs), implemented in PyTorch and HuggingFace, with a focus on simplicity of code implementations, extensibility of new features, and reproducibility of training results.
+
+- With TinyLLaVA Factory, you can customize your own large multimodal models with less coding effort and less coding mistakes.
+
+- TinyLLaVA Factory integrates a suite of cutting-edge models and methods. 
+
+  - LLM currently supports **OpenELM**, **TinyLlama**, **StableLM**, **Qwen**, **Gemma**, and **Phi**. 
+
+  - Vision tower currently supports **CLIP,** **SigLIP**, **Dino**, and **combination of CLIP and Dino**.
+    
+  - Connector currently supports **MLP**, **Qformer**, and **Resampler**.
+    
+  - Training Recipe currently supports **Frozen/Fully/Partially tuning** and **LoRA/QLoRA tuning**.
+  
+- The password to access our demo is '1234'.
+
+## Contents
+
+- [🎉 News](#-news)
+- [🔥 Takeaways](#-takeaways)
+- [Contents](#contents)
+- [Installation and Requirements](#installation-and-requirements)
+    - [Upgrade to the latest code base](#upgrade-to-the-latest-code-base)
+- [Get Started](#get-started)
+    - [1. Data Preparation](#1-data-preparation)
+    - [2. Train](#2-train)
+    - [3. Evaluation](#3-evaluation)
+- [Model Zoo](#model-zoo)
+  - [Trained Models](#trained-models)
+    - [Model Performance](#model-performance)
+  - [Legacy Models](#legacy-models)
+- [Launch Demo Locally](#launch-demo-locally)
+  - [Gradio Web Demo](#gradio-web-demo)
+  - [CLI Inference](#cli-inference)
+  - [Quick Inference Scripts](#quick-inference-scripts)
+- [Custom Finetune](#custom-finetune)
+- [Customize Your Own Large Multimodel Models](#customize-your-own-large-multimodel-models)
+  - [LLM](#llm)
+  - [Vision Tower](#vision-tower)
+  - [Connector](#connector)
+- [Acknowledgement](#acknowledgement)
+- [Contact](#contact)
+- [✏ Citation](#-citation)
+- [❤️ Community efforts](#️-community-efforts)
+
+
+## Installation and Requirements
+
+Please note that our environment requirements are different from LLaVA's environment requirements. We strongly recommend you create the environment from scratch as follows.
+
+1. Clone this repository and navigate to the folder
+```bash
+git clone https://github.com/TinyLLaVA/TinyLLaVA_Factory.git
+cd TinyLLaVA_Factory
+```
+
+2. Create a conda environment, activate it and install Packages
+```Shell
+conda create -n tinyllava_factory python=3.10 -y
+conda activate tinyllava_factory
+pip install --upgrade pip  # enable PEP 660 support
+pip install -e .
+```
+
+3. Install additional packages
+```Shell
+pip install flash-attn --no-build-isolation
+```
+#### Upgrade to the latest code base
+
+```Shell
+git pull
+pip install -e .
+```
+
+## Get Started
+
+#### 1. Data Preparation
+
+Please refer to the [Data Preparation](https://tinyllava-factory.readthedocs.io/en/latest/Prepare%20Datasets.html) section in our [Documenation](https://tinyllava-factory.readthedocs.io/en/latest/).
+
+#### 2. Train
+
+Here's an example for training a LMM using Phi-2.
+
+- Replace data paths with yours in `scripts/train/train_phi.sh`
+- Replace `output_dir` with yours in `scripts/train/pretrain.sh`
+- Replace `pretrained_model_path` and `output_dir` with yours in `scripts/train/finetune.sh`
+- Adjust your GPU ids (localhost) and `per_device_train_batch_size` in `scripts/train/pretrain.sh` and `scripts/train/finetune.sh`
+
+```bash
+bash scripts/train/train_phi.sh
+```
+
+Important hyperparameters used in pretraining and finetuning are provided below.
+
+| Training Stage | Global Batch Size | Learning rate | conv_version |
+| -------------- | :---------------: | :-----------: | :----------: |
+| Pretraining    | 256               | 1e-3          | pretrain     |
+| Finetuning     | 128               | 2e-5          | phi          |
+
+**Tips:** 
+
+Global Batch Size = num of GPUs * `per_device_train_batch_size` * `gradient_accumulation_steps`, we recommand you always keep global batch size and learning rate as above except for lora tuning your model.
+
+`conv_version` is a hyperparameter used for choosing different chat templates for different LLMs. In the pretraining stage, `conv_version` is the same for all LLMs, using `pretrain`. In the finetuning stage, we use
+
+`phi` for Phi-2, StableLM, Qwen-1.5
+
+`llama` for TinyLlama, OpenELM
+
+`gemma` for Gemma
+
+#### 3. Evaluation
+
+Please refer to the [Evaluation](https://tinyllava-factory.readthedocs.io/en/latest/Evaluation.html) section in our [Documenation](https://tinyllava-factory.readthedocs.io/en/latest/Evaluation.html).
+
+## Model Zoo
+
+### Trained Models
+
+which are trained using TinyLLaVA Factory.
+
+- [TinyLLaVA-Phi-2-SigLIP-3.1B](https://huggingface.co/tinyllava/TinyLLaVA-Phi-2-SigLIP-3.1B)
+- [TinyLLaVA-Gemma-SigLIP-2.4B](https://huggingface.co/tinyllava/TinyLLaVA-Gemma-SigLIP-2.4B)
+- [TinyLLaVA-OpenELM-450M-SigLIP-0.89B](https://huggingface.co/jiajunlong/TinyLLaVA-0.89B)
+- [TinyLLaVA-Qwen2-0.5B-SigLIP](https://huggingface.co/Zhang199/TinyLLaVA-Qwen2-0.5B-SigLIP)
+- [TinyLLaVA-Qwen2.5-3B-SigLIP](https://huggingface.co/Zhang199/TinyLLaVA-Qwen2.5-3B-SigLIP)
+
+#### Model Performance
+
+| VT (HF Path)                      | LLM (HF Path)                      | Recipe    | VQA-v2 | GQA  | SQA-image | TextVQA | MM-Vet | POPE | MME    | MMMU-val |
+| --------------------------------- | ---------------------------------- | --------- | :----: | :--: | :-------: | :-----: | :----: | :--: | :----: | :------: |
+| openai/clip-vit-large-patch14-336 | apple/OpenELM-450M-Instruct        | base      | 69.5   | 52.1 | 50.6      | 40.4    | 20.0   | 83.6 | 1052.9 | 23.9     |
+| google/siglip-so400m-patch14-384  | apple/OpenELM-450M-Instruct        | base      | 71.7   | 53.9 | 54.1      | 44.0    | 20.0   | 85.4 | 1118.8 | 24.0     |
+| google/siglip-so400m-patch14-384  | Qwen/Qwen2-0.5B                    | base      | 72.3   | 55.8 | 60.1      | 45.2    | 19.5   | 86.6 | 1153.0 | 29.7     |
+| google/siglip-so400m-patch14-384  | Qwen/Qwen2.5-0.5B                  | base      | 75.3   | 59.5 | 60.3      | 48.3    | 23.9   | 86.1 | 1253.0 | 33.3     |
+| google/siglip-so400m-patch14-384  | Qwen/Qwen2.5-3B                    | base      | 79.4   | 62.5 | 74.1      | 58.3    | 34.8   | 87.4 | 1438.7 | 39.9     |
+| openai/clip-vit-large-patch14-336 | TinyLlama/TinyLlama-1.1B-Chat-v1.0 | base      | 73.7   | 58.0 | 59.9      | 46.3    | 23.2   | 85.5 | 1284.6 | 27.9     |
+| google/siglip-so400m-patch14-384  | TinyLlama/TinyLlama-1.1B-Chat-v1.0 | base      | 75.5   | 58.6 | 64.0      | 49.6    | 23.5   | 86.3 | 1256.5 | 28.3     |
+| openai/clip-vit-large-patch14-336 | stabilityai/stablelm-2-zephyr-1_6b | base      | 75.9   | 59.5 | 64.6      | 50.5    | 27.3   | 86.1 | 1368.1 | 31.8     |
+| google/siglip-so400m-patch14-384  | stabilityai/stablelm-2-zephyr-1_6b | base      | 78.2   | 60.7 | 66.7      | 56.0    | 29.4   | 86.3 | 1319.3 | 32.6     |
+| google/siglip-so400m-patch14-384  | google/gemma-2b-it                 | base      | 78.4   | 61.6 | 64.4      | 53.6    | 26.9   | 86.4 | 1339.0 | 31.7     |
+| openai/clip-vit-large-patch14-336 | microsoft/phi-2                    | base      | 76.8   | 59.4 | 71.2      | 53.4    | 31.7   | 86.8 | 1448.6 | 36.3     |
+| google/siglip-so400m-patch14-384  | microsoft/phi-2                    | base      | 79.2   | 61.6 | 71.9      | 57.4    | 35.0   | 87.2 | 1462.4 | 38.2     |
+| google/siglip-so400m-patch14-384  | microsoft/phi-2                    | base&lora | 77.6   | 59.7 | 71.6      | 53.8    | 33.3   | 87.9 | 1413.2 | 35.6     |
+| google/siglip-so400m-patch14-384  | microsoft/phi-2                    | share     | 80.1   | 62.1 | 73.0      | 60.3    | 37.5   | 87.2 | 1466.4 | 38.4     |
+
+### Legacy Models
+
+which are trained using the old codebase TinyLLaVABench.
+
+- [TinyLLaVA-3.1B](https://huggingface.co/bczhou/TinyLLaVA-3.1B)
+- [TinyLLaVA-2.0B](https://huggingface.co/bczhou/TinyLLaVA-2.0B)
+- [TinyLLaVA-1.5B](https://huggingface.co/bczhou/TinyLLaVA-1.5B)
+- [tiny-llava-hf](https://huggingface.co/bczhou/tiny-llava-v1-hf)
+
+If you have models trained by our old codebase TinyLLaVABench and you still want to use them, we provide an example of [TinyLLaVA-3.1B](https://huggingface.co/bczhou/TinyLLaVA-3.1B) for how to use legacy models.
+
+<details>
+<summary>Example of using legacy models</summary>
+
+
+```Python
+from tinyllava.eval.run_tiny_llava import eval_model
+from tinyllava.model.convert_legecy_weights_to_tinyllavafactory import *
+
+model = convert_legecy_weights_to_tinyllavafactory('bczhou/TinyLLaVA-3.1B')
+
+prompt = "What are the things I should be cautious about when I visit here?"
+image_file = "https://llava-vl.github.io/static/images/view.jpg"
+
+args = type('Args', (), {
+    "model_path": None,
+    "model": model,
+    "query": prompt,
+    "conv_mode": "phi", # the same as conv_version in the training stage. Different LLMs have different conv_mode/conv_version, please replace it
+    "image_file": image_file,
+    "sep": ",",
+    "temperature": 0,
+    "top_p": None,
+    "num_beams": 1,
+    "max_new_tokens": 512
+})()
+
+eval_model(args)
+
+"""
+Output: 
+When visiting this serene lakeside location with a wooden dock, there are a few things to be cautious about. First, ensure that the dock is stable and secure before stepping onto it, as it might be slippery or wet, especially if it's a wooden structure. Second, be mindful of the surrounding water, as it can be deep or have hidden obstacles, such as rocks or debris, that could pose a risk. Additionally, be aware of the weather conditions, as sudden changes in weather can make the area more dangerous. Lastly, respect the natural environment and wildlife, and avoid littering or disturbing the ecosystem.
+"""
+```
+
+</details>
+
+
+
+## Launch Demo Locally
+
+### Gradio Web Demo
+Launch a local web demo by running:
+```bash
+python tinyllava/serve/app.py --model-path tinyllava/TinyLLaVA-Phi-2-SigLIP-3.1B
+```
+### CLI Inference
+We also support running inference with CLI. To use our model, run:
+```bash
+python -m tinyllava.serve.cli \
+   --model-path tinyllava/TinyLLaVA-Phi-2-SigLIP-3.1B \
+   --image-file "./tinyllava/serve/examples/extreme_ironing.jpg" 
+```
+### Quick Inference Scripts
+If you want to launch the model trained by yourself or us locally, here's an example.
+<details>
+<summary>Run inference with the model trained by yourself</summary>
+
+```Python
+from tinyllava.eval.run_tiny_llava import eval_model
+
+model_path = "/absolute/path/to/your/model/"
+prompt = "What are the things I should be cautious about when I visit here?"
+image_file = "https://llava-vl.github.io/static/images/view.jpg"
+conv_mode = "phi" # or llama, gemma, etc
+
+args = type('Args', (), {
+    "model_path": model_path,
+    "model": None,
+    "query": prompt,
+    "conv_mode": conv_mode,
+    "image_file": image_file,
+    "sep": ",",
+    "temperature": 0,
+    "top_p": None,
+    "num_beams": 1,
+    "max_new_tokens": 512
+})()
+
+eval_model(args)
+
+"""
+Output: 
+XXXXXXXXXXXXXXXXX
+"""
+```
+</details>
+
+<details>
+<summary>Run inference with the model trained by us using huggingface transformers</summary>
+
+```Python
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+hf_path = 'tinyllava/TinyLLaVA-Phi-2-SigLIP-3.1B'
+model = AutoModelForCausalLM.from_pretrained(hf_path, trust_remote_code=True)
+model.cuda()
+config = model.config
+tokenizer = AutoTokenizer.from_pretrained(hf_path, use_fast=False, model_max_length = config.tokenizer_model_max_length,padding_side = config.tokenizer_padding_side)
+prompt="What are these?"
+image_url="http://images.cocodataset.org/val2017/000000039769.jpg"
+output_text, genertaion_time = model.chat(prompt=prompt, image=image_url, tokenizer=tokenizer)
+
+print('model output:', output_text)
+print('runing time:', genertaion_time)
+```
+</details>
+
+## Custom Finetune
+If you want to finetune TinyLLaVA with your custom datasets, please refer to [here](https://github.com/TinyLLaVA/TinyLLaVA_Factory/blob/main/CUSTOM_FINETUNE.md).
+
+## Customize Your Own Large Multimodel Models
+
+### LLM
+
+If you want to add a new LLM by yourself, you need to create two files: one for chat template and the other for language model, under the folders `tinyllava/data/template/` and `tinyllava/model/llm/`.
+
+Here is an example of adding the Gemma model.
+
+Firstly, create `tinyllava/data/template/gemma_template.py`, which will be used for the finetuning stage.
+
+```python
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Tuple, Union
+from packaging import version
+
+from .formatter import EmptyFormatter, StringFormatter
+from .base import Template
+from .formatter import Formatter
+from . import register_template
+from ...utils.constants import *
+
+from transformers import PreTrainedTokenizer
+import torch
+import tokenizers
+
+    
+system = "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions."
+
+@register_template('gemma') # Enable the TemplateFactory to obtain the added template by this string ('gemma').
+@dataclass
+class GemmaTemplate(Template):
+    format_image_token: "Formatter" = StringFormatter(slot="<image>\n{{content}}")
+    format_user: "Formatter" = StringFormatter(slot="USER" + ": " + "{{content}}" + " ")
+    format_assistant: "Formatter" = StringFormatter(slot="ASSISTANT" + ": " + "{{content}}" + "<eos>") # to be modified according to the tokenizer you choose
+    system: "Formatter" = EmptyFormatter(slot=system+" ")
+    separator: "Formatter" = EmptyFormatter(slot=[' ASSISTANT: ', '<eos>']) # to be modified according to the tokenizer you choose
+
+    def _make_masks(self, labels, tokenizer, sep, eos_token_length, rounds):
+        # your code here
+        return labels, cur_len
+```
+**Tips:**
+
+Please ensure that the `labels` (returned by the `_make_masks` function) follows this format: answers and the eos token id are not masked, and the other tokens are masked with `-100`.
+
+Secondly, create `tinyllava/model/llm/gemma.py`.
+
+```python
+from transformers import GemmaForCausalLM, AutoTokenizer
+# The LLM you want to add along with its corresponding tokenizer.
+
+from . import register_llm
+
+# Add GemmaForCausalLM along with its corresponding tokenizer and handle special tokens.
+@register_llm('gemma') # Enable the LLMFactory to obtain the added LLM by this string ('gemma').
+def return_gemmaclass(): 
+    def tokenizer_and_post_load(tokenizer):
+        tokenizer.pad_token = tokenizer.unk_token
+        return tokenizer
+    return (GemmaForCausalLM, (AutoTokenizer, tokenizer_and_post_load))
+```
+
+Finally, create `scripts/train/train_gemma.sh` with the corresponding `LLM_VERSION` and `CONV_VERSION`.
+
+### Vision Tower
+
+If you want to add a new vision tower, you need to implement a new vision tower class that should be inherited from the base class `VisionTower`. Here's an example of the MoF vision tower.
+
+First, create `tinyllava/model/vision_tower/mof.py`
+
+```python
+@register_vision_tower('mof')      
+class MoFVisionTower(VisionTower):
+    def __init__(self, cfg):
+        super().__init__(cfg)
+
+        self._vision_tower = MoF(cfg)
+        self._image_processor = # your image processor
+  
+    def _load_model(self, vision_tower_name, **kwargs):
+        # your code here, make sure your model can be correctly loaded from pretrained parameters either by huggingface or pytorch loading
+
+    def forward(self, x, **kwargs):
+        # your code here
+```
+
+Then, modify your training scripts with the corresponding `VT_VERSION`.
+
+### Connector
+
+If you want to add a new connector, you need to implement a new connector class that should be inherited from the base class `Connector`. Here's an example of the Linear connector.
+
+First, create `tinyllava/model/connector/linear.py`
+
+
+```python
+import torch.nn as nn
+
+from . import register_connector
+from .base import Connector
+    
+@register_connector('linear') #Enable the ConnectorMFactory to obtain the added connector by this string ('linear').     
+class LinearConnector(Connector):
+    def __init__(self, config):
+        super().__init__()
+        self._connector =  nn.Linear(config.vision_hidden_size, config.hidden_size) # define your connector model
+```
+
+Then, modify your training scripts with the corresponding `CN_VERSION`.
+
+## Acknowledgement
+We give special thanks to Lei Zhao, Luche Wang, Kaijun Luo, and Junchen Wang for building the [Demo](http://8843843nmph5.vicp.fun/#/).
+
+## Contact
+If you have any questions, feel free to either initiate an *Issue* or contact us by WeChat (WeChatID: *TinyLLaVA*).
+
+## &#x270F; Citation
+
+If you find our paper and code useful in your research, please consider giving a star :star: and citation :pencil:.
+
+```BibTeX
+@misc{zhou2024tinyllava,
+      title={TinyLLaVA: A Framework of Small-scale Large Multimodal Models}, 
+      author={Baichuan Zhou and Ying Hu and Xi Weng and Junlong Jia and Jie Luo and Xien Liu and Ji Wu and Lei Huang},
+      year={2024},
+      eprint={2402.14289},
+      archivePrefix={arXiv},
+      primaryClass={cs.LG}
+}
+```
+```BibTeX
+@article{jia2024tinyllava,
+  title={TinyLLaVA Factory: A Modularized Codebase for Small-scale Large Multimodal Models},
+  author={Jia, Junlong and Hu, Ying and Weng, Xi and Shi, Yiming and Li, Miao and Zhang, Xingjian and Zhou, Baichuan and Liu, Ziyu and Luo, Jie and Huang, Lei and Wu, Ji},
+  journal={arXiv preprint arXiv:2405.11788},
+  year={2024}
+}
+```
+
+
+## ❤️ Community efforts
+* Our codebase is built upon the [LLaVA](https://github.com/haotian-liu/LLaVA) project. Great work!
+* Our project uses data from the [ShareGPT4V](https://github.com/InternLM/InternLM-XComposer/tree/main/projects/ShareGPT4V) project. Great work!

TinyLLaVA_Factory/pyproject.toml

@@ -0,0 +1,38 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "tinyllava"
+version = "1.0.0"
+description = "A Framework of Small-scale Large Multimodal Models."
+readme = "README.md"
+requires-python = ">=3.9"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: Apache Software License",
+]
+dependencies = [
+    "torch==2.0.1", "torchvision==0.15.2", "tiktoken", "openpyxl", "tensorboardX",
+    "transformers==4.40.1", "tokenizers==0.19.0", "sentencepiece==0.1.99", "shortuuid",
+    "accelerate==0.27.2", "bitsandbytes==0.41.0", "peft==0.10.0",
+    "pydantic<2,>=1", "markdown2[all]", "numpy==1.26.4", "scikit-learn==1.2.2",
+    "gradio==3.35.2", "gradio_client==0.2.9",
+    "requests", "httpx==0.24.0", "uvicorn", "fastapi",
+    "einops==0.6.1", "einops-exts==0.0.4", "timm==0.6.13",
+    "deepspeed==0.14.0", "ninja", "wandb",
+]
+
+[project.optional-dependencies]
+train = ["deepspeed==0.14.0", "ninja", "wandb"]
+
+[project.urls]
+"Homepage" = "https://github.com/DLCV-BUAA/TinyLLaVABench"
+"Bug Tracker" = "https://github.com/DLCV-BUAA/TinyLLaVABench/issues"
+
+[tool.setuptools.packages.find]
+exclude = ["assets*", "benchmark*", "docs", "dist*", "playground*", "scripts*", "tests*"]
+
+[tool.wheel]
+exclude = ["assets*", "benchmark*", "docs", "dist*", "playground*", "scripts*", "tests*"]
+

TinyLLaVA_Factory/scripts/convert_answer_to_mmmu.py

@@ -0,0 +1,31 @@
+import argparse
+import json
+import os
+
+
+def eval_model(args):
+    answers = [json.loads(q) for q in open(os.path.expanduser(args.answers_file), "r")]
+    answers_dict = {}
+    for answer in answers:
+        answers_dict[answer["question_id"]] = answer["text"]
+        # print(answer)
+
+    with open(args.answers_output, "w") as f:
+        json.dump(answers_dict, f)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--answers-file",
+        type=str,
+        required=True
+    )
+    parser.add_argument(
+        "--answers-output",
+        type=str,
+        required=True
+    )
+    args = parser.parse_args()
+
+    eval_model(args)

TinyLLaVA_Factory/scripts/convert_gqa_for_eval.py

@@ -0,0 +1,18 @@
+import os
+import json
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--src", type=str)
+parser.add_argument("--dst", type=str)
+args = parser.parse_args()
+
+all_answers = []
+for line_idx, line in enumerate(open(args.src)):
+    res = json.loads(line)
+    question_id = res['question_id']
+    text = res['text'].rstrip('.').lower()
+    all_answers.append({"questionId": question_id, "prediction": text})
+
+with open(args.dst, 'w') as f:
+    json.dump(all_answers, f)

TinyLLaVA_Factory/scripts/convert_mmvet_for_eval.py

@@ -0,0 +1,18 @@
+import os
+import json
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--src", type=str)
+parser.add_argument("--dst", type=str)
+args = parser.parse_args()
+
+cur_result = {}
+
+for line in open(args.src):
+    data = json.loads(line)
+    qid = data['question_id']
+    cur_result[f'v1_{qid}'] = data['text']
+
+with open(args.dst, 'w') as f:
+    json.dump(cur_result, f, indent=2)

TinyLLaVA_Factory/scripts/convert_vqav2_for_submission.py

@@ -0,0 +1,56 @@
+import os
+import argparse
+import json
+
+from tinyllava.eval.m4c_evaluator import EvalAIAnswerProcessor
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--dir', type=str, default="./playground/data/eval/vqav2")
+    parser.add_argument('--ckpt', type=str, required=True)
+    parser.add_argument('--split', type=str, required=True)
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+
+    args = parse_args()
+
+    src = os.path.join(args.dir, 'answers', args.split, args.ckpt, 'merge.jsonl')
+    test_split = os.path.join(args.dir, 'llava_vqav2_mscoco_test2015.jsonl')
+    dst = os.path.join(args.dir, 'answers_upload', args.split, f'{args.ckpt}.json')
+    os.makedirs(os.path.dirname(dst), exist_ok=True)
+
+    results = []
+    error_line = 0
+    for line_idx, line in enumerate(open(src)):
+        try:
+            results.append(json.loads(line))
+        except:
+            error_line += 1
+
+    results = {x['question_id']: x['text'] for x in results}
+    test_split = [json.loads(line) for line in open(test_split)]
+    split_ids = set([x['question_id'] for x in test_split])
+
+    print(f'total results: {len(results)}, total split: {len(test_split)}, error_line: {error_line}')
+
+    all_answers = []
+
+    answer_processor = EvalAIAnswerProcessor()
+
+    for x in test_split:
+        if x['question_id'] not in results:
+            all_answers.append({
+                'question_id': x['question_id'],
+                'answer': ''
+            })
+        else:
+            all_answers.append({
+                'question_id': x['question_id'],
+                'answer': answer_processor(results[x['question_id']])
+            })
+
+    with open(dst, 'w') as f:
+        json.dump(all_answers, open(dst, 'w'))

TinyLLaVA_Factory/scripts/eval/gqa.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+gpu_list="${CUDA_VISIBLE_DEVICES:-0}"
+IFS=',' read -ra GPULIST <<< "$gpu_list"
+
+CHUNKS=${#GPULIST[@]}
+
+SPLIT="llava_gqa_testdev_balanced"
+GQADIR="/home/ai/data/llava/dataset/eval/gqa"
+
+MODEL_PATH="/mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-phi-2-siglip-so400m-patch14-384-base-finetune/"
+MODEL_NAME="tiny-llava-phi-2-siglip-so400m-patch14-384-base-finetune"
+EVAL_DIR="/home/ai/data/llava/dataset/eval"
+
+for IDX in $(seq 0 $((CHUNKS-1))); do
+    CUDA_VISIBLE_DEVICES=${GPULIST[$IDX]} python -m tinyllava.eval.model_vqa_loader \
+        --model-path $MODEL_PATH \
+        --question-file $EVAL_DIR/gqa/$SPLIT.jsonl \
+        --image-folder $EVAL_DIR/gqa/images \
+        --answers-file $EVAL_DIR/gqa/answers/$SPLIT/$MODEL_NAME/${CHUNKS}_${IDX}.jsonl \
+        --num-chunks $CHUNKS \
+        --chunk-idx $IDX \
+        --temperature 0 \
+        --conv-mode phi &
+done
+
+wait
+
+output_file=$EVAL_DIR/gqa/answers/$SPLIT/$MODEL_NAME/merge.jsonl
+
+# Clear out the output file if it exists.
+> "$output_file"
+
+# Loop through the indices and concatenate each file.
+for IDX in $(seq 0 $((CHUNKS-1))); do
+    cat $EVAL_DIR/gqa/answers/$SPLIT/$MODEL_NAME/${CHUNKS}_${IDX}.jsonl >> "$output_file"
+done
+
+python scripts/convert_gqa_for_eval.py --src $output_file --dst $GQADIR/testdev_balanced_predictions.json
+
+cd $GQADIR
+python eval/eval.py --tier testdev_balanced
+

TinyLLaVA_Factory/scripts/eval/mme.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+MODEL_PATH="/home/jiajunlong/LLaVA/ying/checkpoints/tiny-llava-TinyLlama-1.1B-Chat-v1.0-clip-vit-large-patch14-336-tinyllama-llava-finetune"
+MODEL_NAME="tiny-llava-TinyLlama-1.1B-Chat-v1.0-clip-vit-large-patch14-336-tinyllama-llava-finetune"
+EVAL_DIR="/home/jiajunlong/llava_data/eval"
+
+python -m tinyllava.eval.model_vqa_loader \
+    --model-path $MODEL_PATH \
+    --question-file $EVAL_DIR/MME/llava_mme.jsonl \
+    --image-folder $EVAL_DIR/MME/MME_Benchmark_release_version \
+    --answers-file $EVAL_DIR/MME/answers/$MODEL_NAME.jsonl \
+    --temperature 0 \
+   --conv-mode llama
+
+cd $EVAL_DIR/MME
+
+python convert_answer_to_mme.py --experiment $MODEL_NAME
+
+cd eval_tool
+
+python calculation.py --results_dir answers/$MODEL_NAME
+

TinyLLaVA_Factory/scripts/eval/mmmu.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+MODEL_PATH="/mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-phi-2-siglip-so400m-patch14-384-base-finetune-final"
+MODEL_NAME="tiny-llava-phi-2-siglip-so400m-patch14-384-base-finetune-final"
+EVAL_DIR="/home/ai/data/llava/dataset/eval"
+
+python -m tinyllava.eval.model_vqa_mmmu \
+    --model-path $MODEL_PATH \
+    --question-file $EVAL_DIR/MMMU/anns_for_eval.json \
+    --image-folder $EVAL_DIR/MMMU/all_images \
+    --answers-file $EVAL_DIR/MMMU/answers/$MODEL_NAME.jsonl \
+    --temperature 0 \
+    --conv-mode phi
+
+python scripts/convert_answer_to_mmmu.py \
+    --answers-file $EVAL_DIR/MMMU/answers/$MODEL_NAME.jsonl \
+    --answers-output $EVAL_DIR/MMMU/answers/"$MODEL_NAME"_output.json
+
+cd $EVAL_DIR/MMMU/eval
+
+python main_eval_only.py --output_path $EVAL_DIR/MMMU/answers/"$MODEL_NAME"_output.json

TinyLLaVA_Factory/scripts/eval/mmvet.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+MODEL_PATH="/mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-phi-2-clip-vit-large-patch14-336-baseline-finetune/"
+MODEL_NAME="tiny-llava-phi-2-clip-vit-large-patch14-336-baseline-finetune2"
+EVAL_DIR="/home/ai/data/llava/dataset/eval"
+
+python -m tinyllava.eval.model_vqa \
+    --model-path $MODEL_PATH \
+    --question-file $EVAL_DIR/mm-vet/llava-mm-vet.jsonl \
+    --image-folder $EVAL_DIR/mm-vet/images \
+    --answers-file $EVAL_DIR/mm-vet/answers/$MODEL_NAME.jsonl \
+    --temperature 0 \
+    --conv-mode phi
+
+mkdir -p $EVAL_DIR/mm-vet/results
+
+python scripts/convert_mmvet_for_eval.py \
+    --src $EVAL_DIR/mm-vet/answers/$MODEL_NAME.jsonl \
+    --dst $EVAL_DIR/mm-vet/results/$MODEL_NAME.json

TinyLLaVA_Factory/scripts/eval/pope.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+MODEL_PATH="/mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-stablelm-2-zephyr-1_6b-siglip-so400m-patch14-384-base-finetune/"
+MODEL_NAME="tiny-llava-stablelm-2-zephyr-1_6b-siglip-so400m-patch14-384-base-finetune"
+EVAL_DIR="/home/ai/data/llava/dataset/eval"
+
+python -m tinyllava.eval.model_vqa_pope \
+    --model-path $MODEL_PATH \
+    --question-file $EVAL_DIR/pope/llava_pope_test.jsonl \
+    --image-folder $EVAL_DIR/pope/val2014 \
+    --answers-file $EVAL_DIR/pope/answers/$MODEL_NAME.jsonl \
+    --temperature 0 \
+    --conv-mode phi
+
+python tinyllava/eval/eval_pope.py \
+    --annotation-dir $EVAL_DIR/pope/coco \
+    --question-file $EVAL_DIR/pope/llava_pope_test.jsonl \
+    --result-file $EVAL_DIR/pope/answers/$MODEL_NAME.jsonl

TinyLLaVA_Factory/scripts/eval/sqa.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+MODEL_PATH="/mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-phi-2-siglip-so400m-patch14-384-base-finetune"
+MODEL_NAME="tiny-llava-phi-2-siglip-so400m-patch14-384-base-finetune"
+EVAL_DIR="/home/ai/data/llava/dataset/eval"
+python -m tinyllava.eval.model_vqa_science \
+    --model-path $MODEL_PATH \
+    --question-file $EVAL_DIR/scienceqa/llava_test_CQM-A.json \
+    --image-folder $EVAL_DIR/scienceqa/images/test \
+    --answers-file $EVAL_DIR/scienceqa/answers/$MODEL_NAME.jsonl \
+    --single-pred-prompt \
+    --temperature 0 \
+    --conv-mode phi
+
+python tinyllava/eval/eval_science_qa.py \
+    --base-dir $EVAL_DIR/scienceqa \
+    --result-file $EVAL_DIR/scienceqa/answers/$MODEL_NAME.jsonl \
+    --output-file $EVAL_DIR/scienceqa/answers/"$MODEL_NAME"_output.jsonl \
+    --output-result $EVAL_DIR/scienceqa/answers/"$MODEL_NAME"_result.json
+

TinyLLaVA_Factory/scripts/eval/textvqa.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+MODEL_PATH="/mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-phi-2-siglip-so400m-patch14-384-base-finetune/"
+MODEL_NAME="tiny-llava-phi-2-siglip-so400m-patch14-384-base-finetune"
+EVAL_DIR="/home/ai/data/llava/dataset/eval"
+
+python -m tinyllava.eval.model_vqa_loader \
+    --model-path $MODEL_PATH \
+    --question-file $EVAL_DIR/textvqa/llava_textvqa_val_v051_ocr.jsonl \
+    --image-folder $EVAL_DIR/textvqa/train_images \
+    --answers-file $EVAL_DIR/textvqa/answers/$MODEL_NAME.jsonl \
+    --temperature 0 \
+    --conv-mode phi
+
+python -m tinyllava.eval.eval_textvqa \
+    --annotation-file $EVAL_DIR/textvqa/TextVQA_0.5.1_val.json \
+    --result-file $EVAL_DIR/textvqa/answers/$MODEL_NAME.jsonl
+

TinyLLaVA_Factory/scripts/eval/vqav2.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+gpu_list="${CUDA_VISIBLE_DEVICES:-0}"
+IFS=',' read -ra GPULIST <<< "$gpu_list"
+
+CHUNKS=${#GPULIST[@]}
+
+SPLIT="llava_vqav2_mscoco_test-dev2015"
+
+MODEL_PATH="/mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-phi-2-clip-vit-large-patch14-336-baseline-finetune/"
+MODEL_NAME="tiny-llava-phi-2-clip-vit-large-patch14-336-baseline-finetune2"
+EVAL_DIR="/home/ai/data/llava/dataset/eval"
+
+for IDX in $(seq 0 $((CHUNKS-1))); do
+    CUDA_VISIBLE_DEVICES=${GPULIST[$IDX]} python -m tinyllava.eval.model_vqa_loader \
+        --model-path $MODEL_PATH \
+        --question-file $EVAL_DIR/vqav2/$SPLIT.jsonl \
+        --image-folder $EVAL_DIR/vqav2/test2015 \
+        --answers-file $EVAL_DIR/vqav2/answers/$SPLIT/$MODEL_NAME/${CHUNKS}_${IDX}.jsonl \
+        --num-chunks $CHUNKS \
+        --chunk-idx $IDX \
+        --temperature 0 \
+        --conv-mode phi &
+done
+
+wait
+
+output_file=$EVAL_DIR/vqav2/answers/$SPLIT/$MODEL_NAME/merge.jsonl
+
+# Clear out the output file if it exists.
+> "$output_file"
+
+# Loop through the indices and concatenate each file.
+for IDX in $(seq 0 $((CHUNKS-1))); do
+    cat $EVAL_DIR/vqav2/answers/$SPLIT/$MODEL_NAME/${CHUNKS}_${IDX}.jsonl >> "$output_file"
+done
+
+python scripts/convert_vqav2_for_submission.py --split $SPLIT --ckpt $MODEL_NAME --dir $EVAL_DIR/vqav2

TinyLLaVA_Factory/scripts/train/custom_finetune.sh

@@ -0,0 +1,45 @@
+DATA_PATH="/home/ai/data/llava/dataset/text_files/llava_v1_5_mix665k.json"
+IMAGE_PATH="/home/ai/data/llava/dataset"
+MODEL_MAX_LENGTH=3072
+OUTPUT_DIR="/mnt/data/sata/yinghu/checkpoints/llava_factory/custom-finetune-TinyLLaVA-Phi-2-SigLIP-3.1B-lora"
+
+deepspeed --include localhost:0,1,2,3 --master_port 29501 tinyllava/train/custom_finetune.py \
+    --deepspeed ./scripts/zero2.json \
+    --data_path  $DATA_PATH \
+    --image_folder $IMAGE_PATH \
+    --is_multimodal True \
+    --conv_version phi \
+    --mm_vision_select_layer -2 \
+    --image_aspect_ratio square \
+    --fp16 True \
+    --training_recipe lora \
+    --tune_type_llm lora \
+    --tune_type_vision_tower frozen \
+    --tune_vision_tower_from_layer 0 \
+    --tune_type_connector full \
+    --lora_r 128 \
+    --lora_alpha 256 \
+    --group_by_modality_length False \
+    --pretrained_model_path "tinyllava/TinyLLaVA-Phi-2-SigLIP-3.1B" \
+    --output_dir $OUTPUT_DIR \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 8 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 50000 \
+    --save_total_limit 1 \
+    --learning_rate 1e-4 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 False \
+    --model_max_length $MODEL_MAX_LENGTH \
+    --gradient_checkpointing True \
+    --dataloader_num_workers 8 \
+    --lazy_preprocess True \
+    --report_to tensorboard \
+    --tokenizer_use_fast False \
+    --run_name custom-finetune-TinyLLaVA-Phi-2-SigLIP-3.1B-lora

TinyLLaVA_Factory/scripts/train/finetune.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+if [ $# -ne 10 ]; then
+    echo "Usage: $0 <DATA_PATH> <IMAGE_PATH> <LLM_VERSION> <VT_VERSION> <VT_VERSION2> <CN_VERSION> <CONV_VERSION> <VERSION> <TRAIN_RECIPE> <MODEL_MAX_LENGTH>"
+    exit 1
+fi
+
+# Assign the arguments to variables
+DATA_PATH="$1"
+IMAGE_PATH="$2"
+LLM_VERSION="$3"
+VT_VERSION="$4"
+VT_VERSION2="$5"
+CN_VERSION="$6"
+CONV_VERSION="$7"
+VERSION="$8"
+TRAIN_RECIPE="$9"
+MODEL_MAX_LENGTH="${10}"
+
+VT_VARIANT="${VT_VERSION#*/}"
+LLM_VARIANT="${LLM_VERSION#*/}"
+
+deepspeed --include localhost:4,5,6,7 --master_port 29501 tinyllava/train/train.py \
+    --deepspeed ./scripts/zero3.json \
+    --data_path  $DATA_PATH \
+    --image_folder $IMAGE_PATH \
+    --is_multimodal True \
+    --conv_version $CONV_VERSION \
+    --model_name_or_path $LLM_VERSION \
+    --vision_tower $VT_VERSION \
+    --vision_tower2 "$VT_VERSION2" \
+    --connector_type $CN_VERSION \
+    --mm_vision_select_layer -2 \
+    --image_aspect_ratio square \
+    --attn_implementation flash_attention_2 \
+    --fp16 True \
+    --training_recipe $TRAIN_RECIPE \
+    --tune_type_llm full \
+    --tune_type_vision_tower frozen\
+    --tune_vision_tower_from_layer 0 \
+    --tune_type_connector full \
+    --group_by_modality_length True \
+    --pretrained_model_path /mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-pretrain \
+    --output_dir /mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-finetune \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 8 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 4 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 50000 \
+    --save_total_limit 1 \
+    --learning_rate 2e-5 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 False \
+    --model_max_length $MODEL_MAX_LENGTH \
+    --gradient_checkpointing True \
+    --dataloader_num_workers 8 \
+    --lazy_preprocess True \
+    --report_to tensorboard \
+    --tokenizer_use_fast False \
+    --run_name tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-finetune

TinyLLaVA_Factory/scripts/train/gemma/finetune_gemma.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+if [ $# -ne 10 ]; then
+    echo "Usage: $0 <DATA_PATH> <IMAGE_PATH> <LLM_VERSION> <VT_VERSION> <VT_VERSION2> <CN_VERSION> <CONV_VERSION> <VERSION> <TRAIN_RECIPE> <MODEL_MAX_LENGTH>"
+    exit 1
+fi
+
+# Assign the arguments to variables
+DATA_PATH="$1"
+IMAGE_PATH="$2"
+LLM_VERSION="$3"
+VT_VERSION="$4"
+VT_VERSION2="$5"
+CN_VERSION="$6"
+CONV_VERSION="$7"
+VERSION="$8"
+TRAIN_RECIPE="$9"
+MODEL_MAX_LENGTH="${10}"
+
+VT_VARIANT="${VT_VERSION#*/}"
+LLM_VARIANT="${LLM_VERSION#*/}"
+
+deepspeed --include localhost:4,5,6,7 --master_port 29501 tinyllava/train/train.py \
+    --deepspeed ./scripts/zero3.json \
+    --data_path  $DATA_PATH \
+    --image_folder $IMAGE_PATH \
+    --is_multimodal True \
+    --conv_version $CONV_VERSION \
+    --model_name_or_path $LLM_VERSION \
+    --vision_tower $VT_VERSION \
+    --vision_tower2 "$VT_VERSION2" \
+    --connector_type $CN_VERSION \
+    --mm_vision_select_layer -2 \
+    --image_aspect_ratio square \
+    --attn_implementation flash_attention_2 \
+    --fp16 True \
+    --training_recipe $TRAIN_RECIPE \
+    --tune_type_llm full \
+    --tune_type_vision_tower frozen\
+    --tune_vision_tower_from_layer 0 \
+    --tune_type_connector full \
+    --group_by_modality_length True \
+    --pretrained_model_path /mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-pretrain \
+    --output_dir /mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-finetune \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 2 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 16 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 50000 \
+    --save_total_limit 1 \
+    --learning_rate 2e-5 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 False \
+    --model_max_length $MODEL_MAX_LENGTH \
+    --gradient_checkpointing True \
+    --dataloader_num_workers 8 \
+    --lazy_preprocess True \
+    --report_to tensorboard \
+    --tokenizer_use_fast False \
+    --run_name tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-finetune

TinyLLaVA_Factory/scripts/train/gemma/pretrain_gemma.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+if [ $# -ne 9 ]; then
+    echo "Usage: $0 <DATA_PATH> <IMAGE_PATH> <LLM_VERSION> <VT_VERSION> <VT_VERSION2> <CN_VERSION> <VERSION> <TRAIN_RECIPE> <MODEL_MAX_LENGTH>"
+    exit 1
+fi
+
+# Assign the arguments to variables
+DATA_PATH="$1"
+IMAGE_PATH="$2"
+LLM_VERSION="$3"
+VT_VERSION="$4"
+VT_VERSION2="$5"
+CN_VERSION="$6"
+VERSION="$7"
+TRAIN_RECIPE="$8"
+MODEL_MAX_LENGTH="$9"
+
+VT_VARIANT="${VT_VERSION#*/}"
+LLM_VARIANT="${LLM_VERSION#*/}"
+
+deepspeed --include localhost:4,5,6,7 --master_port 29501 tinyllava/train/train.py \
+    --deepspeed ./scripts/zero3.json \
+    --data_path  $DATA_PATH\
+    --image_folder $IMAGE_PATH \
+    --is_multimodal True \
+    --conv_version pretrain \
+    --model_name_or_path $LLM_VERSION \
+    --vision_tower $VT_VERSION \
+    --vision_tower2 "$VT_VERSION2" \
+    --connector_type $CN_VERSION \
+    --mm_vision_select_layer -2 \
+    --image_aspect_ratio square \
+    --attn_implementation flash_attention_2 \
+    --fp16 True \
+    --training_recipe $TRAIN_RECIPE \
+    --tune_type_llm frozen \
+    --tune_type_vision_tower frozen \
+    --tune_vision_tower_from_layer 0 \
+    --tune_type_connector full \
+    --output_dir /mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-pretrain \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 8 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 8 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 24000 \
+    --save_total_limit 1 \
+    --learning_rate 1e-3 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 False \
+    --model_max_length $MODEL_MAX_LENGTH \
+    --gradient_checkpointing True \
+    --dataloader_num_workers 8 \
+    --lazy_preprocess True \
+    --report_to tensorboard \
+    --tokenizer_use_fast False \
+    --run_name tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-pretrain

TinyLLaVA_Factory/scripts/train/gemma/train_gemma.sh

@@ -0,0 +1,17 @@
+DATA_PATH=/home/ai/data/llava/dataset/text_files/blip_laion_cc_sbu_558k.json
+FINETUNE_DATA_PATH=/home/ai/data/llava/dataset/text_files/llava_v1_5_mix665k.json
+IMAGE_PATH=/home/ai/data/llava/dataset/llava/llava_pretrain/images
+FINETUNE_IMAGE_PATH=/home/ai/data/llava/dataset
+
+LLM_VERSION=google/gemma-2b-it
+VT_VERSION=google/siglip-so400m-patch14-384
+VT_VERSION2=""
+CN_VERSION=mlp2x_gelu
+CONV_VERSION=gemma
+VERSION=base
+TRAIN_RECIPE=common
+MODEL_MAX_LENGTH=2048
+
+
+bash scripts/train/gemma/pretrain_gemma.sh "$DATA_PATH" "$IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$VERSION" "$TRAIN_RECIPE" "$MODEL_MAX_LENGTH"
+bash scripts/train/gemma/finetune_gemma.sh "$FINETUNE_DATA_PATH" "$FINETUNE_IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$CONV_VERSION" "$VERSION" "$TRAIN_RECIPE" "$MODEL_MAX_LENGTH"

TinyLLaVA_Factory/scripts/train/lora/finetune_lora.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+if [ $# -ne 10 ]; then
+    echo "Usage: $0 <DATA_PATH> <IMAGE_PATH> <LLM_VERSION> <VT_VERSION> <VT_VERSION2> <CN_VERSION> <CONV_VERSION> <VERSION> <TRAIN_RECIPE> <MODEL_MAX_LENGTH>"
+    exit 1
+fi
+
+# Assign the arguments to variables
+DATA_PATH="$1"
+IMAGE_PATH="$2"
+LLM_VERSION="$3"
+VT_VERSION="$4"
+VT_VERSION2="$5"
+CN_VERSION="$6"
+CONV_VERSION="$7"
+VERSION="$8"
+TRAIN_RECIPE="$9"
+MODEL_MAX_LENGTH="${10}"
+
+VT_VARIANT="${VT_VERSION#*/}"
+LLM_VARIANT="${LLM_VERSION#*/}"
+
+deepspeed --include localhost:2,3,4,5 --master_port 29502 tinyllava/train/train.py \
+    --deepspeed ./scripts/zero2.json \
+    --data_path  $DATA_PATH \
+    --image_folder $IMAGE_PATH \
+    --is_multimodal True \
+    --conv_version $CONV_VERSION \
+    --model_name_or_path $LLM_VERSION \
+    --vision_tower $VT_VERSION \
+    --vision_tower2 "$VT_VERSION2" \
+    --connector_type $CN_VERSION \
+    --mm_vision_select_layer -2 \
+    --image_aspect_ratio square \
+    --attn_implementation flash_attention_2 \
+    --fp16 True \
+    --training_recipe $TRAIN_RECIPE \
+    --tune_type_llm lora \
+    --tune_type_vision_tower frozen \
+    --tune_vision_tower_from_layer 0 \
+    --tune_type_connector full \
+    --lora_r 128 \
+    --lora_alpha 256 \
+    --group_by_modality_length False \
+    --pretrained_model_path /mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-pretrain \
+    --output_dir /mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-finetune \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 8 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 4 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 50000 \
+    --save_total_limit 1 \
+    --learning_rate 2e-4 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 False \
+    --model_max_length $MODEL_MAX_LENGTH \
+    --gradient_checkpointing True \
+    --dataloader_num_workers 8 \
+    --lazy_preprocess True \
+    --report_to tensorboard \
+    --tokenizer_use_fast False \
+    --run_name tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-finetune

TinyLLaVA_Factory/scripts/train/lora/finetune_qlora.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+if [ $# -ne 10 ]; then
+    echo "Usage: $0 <DATA_PATH> <IMAGE_PATH> <LLM_VERSION> <VT_VERSION> <VT_VERSION2> <CN_VERSION> <CONV_VERSION> <VERSION> <TRAIN_RECIPE> <MODEL_MAX_LENGTH>"
+    exit 1
+fi
+
+# Assign the arguments to variables
+DATA_PATH="$1"
+IMAGE_PATH="$2"
+LLM_VERSION="$3"
+VT_VERSION="$4"
+VT_VERSION2="$5"
+CN_VERSION="$6"
+CONV_VERSION="$7"
+VERSION="$8"
+TRAIN_RECIPE="$9"
+MODEL_MAX_LENGTH="${10}"
+
+VT_VARIANT="${VT_VERSION#*/}"
+LLM_VARIANT="${LLM_VERSION#*/}"
+
+deepspeed --include localhost:4,5,6,7 --master_port 29502 tinyllava/train/train.py \
+    --deepspeed ./scripts/zero2.json \
+    --data_path  $DATA_PATH \
+    --image_folder $IMAGE_PATH \
+    --is_multimodal True \
+    --conv_version $CONV_VERSION \
+    --model_name_or_path $LLM_VERSION \
+    --vision_tower $VT_VERSION \
+    --vision_tower2 "$VT_VERSION2" \
+    --connector_type $CN_VERSION \
+    --mm_vision_select_layer -2 \
+    --image_aspect_ratio square \
+    --attn_implementation flash_attention_2 \
+    --fp16 True \
+    --training_recipe $TRAIN_RECIPE \
+    --tune_type_llm qlora \
+    --tune_type_vision_tower frozen \
+    --tune_vision_tower_from_layer 0 \
+    --tune_type_connector full \
+    --lora_r 128 \
+    --lora_alpha 256 \
+    --group_by_modality_length False \
+    --pretrained_model_path /mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-pretrain \
+    --output_dir /mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-finetune \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 8 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 4 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 50000 \
+    --save_total_limit 1 \
+    --learning_rate 2e-4 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 False \
+    --model_max_length $MODEL_MAX_LENGTH \
+    --gradient_checkpointing True \
+    --dataloader_num_workers 8 \
+    --lazy_preprocess True \
+    --report_to tensorboard \
+    --tokenizer_use_fast False \
+    --run_name tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-finetune

TinyLLaVA_Factory/scripts/train/lora/train_phi_lora.sh

@@ -0,0 +1,18 @@
+DATA_PATH=/home/ai/data/llava/dataset/text_files/blip_laion_cc_sbu_558k.json
+FINETUNE_DATA_PATH=/home/ai/data/llava/dataset/text_files/llava_v1_5_mix665k.json
+IMAGE_PATH=/home/ai/data/llava/dataset/llava/llava_pretrain/images
+FINETUNE_IMAGE_PATH=/home/ai/data/llava/dataset
+
+LLM_VERSION=microsoft/phi-2
+VT_VERSION=google/siglip-so400m-patch14-384
+VT_VERSION2=""
+CN_VERSION=mlp2x_gelu
+CONV_VERSION=phi
+VERSION=base-lora-zero2-r128
+PRETRAIN_TRAIN_RECIPE=common
+FINETUNE_TRAIN_RECIPE=lora
+MODEL_MAX_LENGTH=3072
+
+
+bash scripts/train/pretrain.sh "$DATA_PATH" "$IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$VERSION" "$PRETRAIN_TRAIN_RECIPE" "$MODEL_MAX_LENGTH"
+bash scripts/train/lora/finetune_lora.sh "$FINETUNE_DATA_PATH" "$FINETUNE_IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$CONV_VERSION" "$VERSION" "$FINETUNE_TRAIN_RECIPE" "$MODEL_MAX_LENGTH"

TinyLLaVA_Factory/scripts/train/lora/train_phi_qlora.sh

@@ -0,0 +1,18 @@
+DATA_PATH=/home/ai/data/llava/dataset/text_files/blip_laion_cc_sbu_558k.json
+FINETUNE_DATA_PATH=/home/ai/data/llava/dataset/text_files/llava_v1_5_mix665k.json
+IMAGE_PATH=/home/ai/data/llava/dataset/llava/llava_pretrain/images
+FINETUNE_IMAGE_PATH=/home/ai/data/llava/dataset
+
+LLM_VERSION=microsoft/phi-2
+VT_VERSION=google/siglip-so400m-patch14-384
+VT_VERSION2=""
+CN_VERSION=mlp2x_gelu
+CONV_VERSION=phi
+VERSION=base-qlora
+PRETRAIN_TRAIN_RECIPE=common
+FINETUNE_TRAIN_RECIPE=qlora_int8
+MODEL_MAX_LENGTH=3072
+
+
+bash scripts/train/pretrain.sh "$DATA_PATH" "$IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$VERSION" "$PRETRAIN_TRAIN_RECIPE" "$MODEL_MAX_LENGTH"
+bash scripts/train/lora/finetune_qlora.sh "$FINETUNE_DATA_PATH" "$FINETUNE_IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$CONV_VERSION" "$VERSION" "$FINETUNE_TRAIN_RECIPE" "$MODEL_MAX_LENGTH"

TinyLLaVA_Factory/scripts/train/openelm/finetune_openelm.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+if [ $# -ne 10 ]; then
+    echo "Usage: $0 <DATA_PATH> <IMAGE_PATH> <LLM_VERSION> <VT_VERSION> <VT_VERSION2> <CN_VERSION> <CONV_VERSION> <VERSION> <TRAIN_RECIPE> <MODEL_MAX_LENGTH>"
+    exit 1
+fi
+
+# Assign the arguments to variables
+DATA_PATH="$1"
+IMAGE_PATH="$2"
+LLM_VERSION="$3"
+VT_VERSION="$4"
+VT_VERSION2="$5"
+CN_VERSION="$6"
+CONV_VERSION="$7"
+VERSION="$8"
+TRAIN_RECIPE="$9"
+MODEL_MAX_LENGTH="${10}"
+
+VT_VARIANT="${VT_VERSION#*/}"
+LLM_VARIANT="${LLM_VERSION#*/}"
+
+deepspeed --include localhost:2,3,4,5 --master_port 29503 tinyllava/train/train.py \
+    --deepspeed ./scripts/zero3.json \
+    --data_path  $DATA_PATH \
+    --image_folder $IMAGE_PATH \
+    --is_multimodal True \
+    --conv_version $CONV_VERSION \
+    --model_name_or_path $LLM_VERSION \
+    --vision_tower $VT_VERSION \
+    --vision_tower2 "$VT_VERSION2" \
+    --connector_type $CN_VERSION \
+    --mm_vision_select_layer -2 \
+    --image_aspect_ratio square \
+    --fp16 True \
+    --tokenizer_name_or_path meta-llama/Llama-2-7b-hf \
+    --training_recipe $TRAIN_RECIPE \
+    --tune_type_llm full \
+    --tune_type_vision_tower frozen\
+    --tune_vision_tower_from_layer 0 \
+    --tune_type_connector full \
+    --group_by_modality_length True \
+    --pretrained_model_path /mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-pretrain \
+    --output_dir /mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-finetune \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 16 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 2 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 50000 \
+    --save_total_limit 1 \
+    --learning_rate 2e-5 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 False \
+    --model_max_length $MODEL_MAX_LENGTH \
+    --gradient_checkpointing True \
+    --dataloader_num_workers 8 \
+    --lazy_preprocess True \
+    --report_to tensorboard \
+    --tokenizer_use_fast False \
+    --run_name tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-finetune

TinyLLaVA_Factory/scripts/train/openelm/pretrain_openelm.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+if [ $# -ne 9 ]; then
+    echo "Usage: $0 <DATA_PATH> <IMAGE_PATH> <LLM_VERSION> <VT_VERSION> <VT_VERSION2> <CN_VERSION> <VERSION> <TRAIN_RECIPE> <MODEL_MAX_LENGTH>"
+    exit 1
+fi
+
+# Assign the arguments to variables
+DATA_PATH="$1"
+IMAGE_PATH="$2"
+LLM_VERSION="$3"
+VT_VERSION="$4"
+VT_VERSION2="$5"
+CN_VERSION="$6"
+VERSION="$7"
+TRAIN_RECIPE="$8"
+MODEL_MAX_LENGTH="$9"
+
+VT_VARIANT="${VT_VERSION#*/}"
+LLM_VARIANT="${LLM_VERSION#*/}"
+
+deepspeed --include localhost:0,1 --master_port 29503 tinyllava/train/train.py \
+    --deepspeed ./scripts/zero3.json \
+    --data_path  $DATA_PATH\
+    --image_folder $IMAGE_PATH \
+    --is_multimodal True \
+    --conv_version pretrain \
+    --model_name_or_path $LLM_VERSION \
+    --vision_tower $VT_VERSION \
+    --vision_tower2 "$VT_VERSION2" \
+    --connector_type $CN_VERSION \
+    --mm_vision_select_layer -2 \
+    --image_aspect_ratio square \
+    --fp16 True \
+    --tokenizer_name_or_path meta-llama/Llama-2-7b-hf \
+    --training_recipe $TRAIN_RECIPE \
+    --tune_type_llm frozen \
+    --tune_type_vision_tower frozen \
+    --tune_vision_tower_from_layer 0 \
+    --tune_type_connector full \
+    --output_dir /mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-pretrain \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 64 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 2 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 24000 \
+    --save_total_limit 1 \
+    --learning_rate 1e-3 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 False \
+    --model_max_length $MODEL_MAX_LENGTH \
+    --gradient_checkpointing True \
+    --dataloader_num_workers 8 \
+    --lazy_preprocess True \
+    --report_to tensorboard \
+    --tokenizer_use_fast False \
+    --run_name tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-pretrain

TinyLLaVA_Factory/scripts/train/openelm/train_openelm.sh

@@ -0,0 +1,17 @@
+DATA_PATH=/home/ai/data/llava/dataset/text_files/blip_laion_cc_sbu_558k.json
+FINETUNE_DATA_PATH=/home/ai/data/llava/dataset/text_files/llava_v1_5_mix665k.json
+IMAGE_PATH=/home/ai/data/llava/dataset/llava/llava_pretrain/images
+FINETUNE_IMAGE_PATH=/home/ai/data/llava/dataset
+
+LLM_VERSION=apple/OpenELM-270M-Instruct
+VT_VERSION=google/siglip-so400m-patch14-384
+VT_VERSION2=""
+CN_VERSION=mlp2x_gelu
+CONV_VERSION=llama
+VERSION=elm_base
+TRAIN_RECIPE=common
+MODEL_MAX_LENGTH=2048
+
+
+bash scripts/train/openelm/pretrain_openelm.sh "$DATA_PATH" "$IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$VERSION" "$TRAIN_RECIPE" "$MODEL_MAX_LENGTH"
+bash scripts/train/openelm/finetune_openelm.sh "$FINETUNE_DATA_PATH" "$FINETUNE_IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$CONV_VERSION" "$VERSION" "$TRAIN_RECIPE" "$MODEL_MAX_LENGTH"

TinyLLaVA_Factory/scripts/train/pretrain.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+if [ $# -ne 9 ]; then
+    echo "Usage: $0 <DATA_PATH> <IMAGE_PATH> <LLM_VERSION> <VT_VERSION> <VT_VERSION2> <CN_VERSION> <VERSION> <TRAIN_RECIPE> <MODEL_MAX_LENGTH>"
+    exit 1
+fi
+
+# Assign the arguments to variables
+DATA_PATH="$1"
+IMAGE_PATH="$2"
+LLM_VERSION="$3"
+VT_VERSION="$4"
+VT_VERSION2="$5"
+CN_VERSION="$6"
+VERSION="$7"
+TRAIN_RECIPE="$8"
+MODEL_MAX_LENGTH="$9"
+
+VT_VARIANT="${VT_VERSION#*/}"
+LLM_VARIANT="${LLM_VERSION#*/}"
+
+deepspeed --include localhost:4,5,6,7 --master_port 29501 tinyllava/train/train.py \
+    --deepspeed ./scripts/zero3.json \
+    --data_path  $DATA_PATH\
+    --image_folder $IMAGE_PATH \
+    --is_multimodal True \
+    --conv_version pretrain \
+    --model_name_or_path $LLM_VERSION \
+    --vision_tower $VT_VERSION \
+    --vision_tower2 "$VT_VERSION2" \
+    --connector_type $CN_VERSION \
+    --mm_vision_select_layer -2 \
+    --image_aspect_ratio square \
+    --attn_implementation flash_attention_2 \
+    --fp16 True \
+    --training_recipe $TRAIN_RECIPE \
+    --tune_type_llm frozen \
+    --tune_type_vision_tower frozen \
+    --tune_vision_tower_from_layer 0 \
+    --tune_type_connector full \
+    --output_dir /mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-pretrain \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 32 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 2 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 24000 \
+    --save_total_limit 1 \
+    --learning_rate 1e-3 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 False \
+    --model_max_length $MODEL_MAX_LENGTH \
+    --gradient_checkpointing True \
+    --dataloader_num_workers 8 \
+    --lazy_preprocess True \
+    --report_to tensorboard \
+    --tokenizer_use_fast False \
+    --run_name tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-pretrain

TinyLLaVA_Factory/scripts/train/qwen2/finetune_qwen2.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+if [ $# -ne 10 ]; then
+    echo "Usage: $0 <DATA_PATH> <IMAGE_PATH> <LLM_VERSION> <VT_VERSION> <VT_VERSION2> <CN_VERSION> <CONV_VERSION> <VERSION> <TRAIN_RECIPE> <MODEL_MAX_LENGTH>"
+    exit 1
+fi
+
+# Assign the arguments to variables
+DATA_PATH="$1"
+IMAGE_PATH="$2"
+LLM_VERSION="$3"
+VT_VERSION="$4"
+VT_VERSION2="$5"
+CN_VERSION="$6"
+CONV_VERSION="$7"
+VERSION="$8"
+TRAIN_RECIPE="$9"
+MODEL_MAX_LENGTH="${10}"
+
+VT_VARIANT="${VT_VERSION#*/}"
+LLM_VARIANT="${LLM_VERSION#*/}"
+
+deepspeed --include localhost:0,1,2,3,4,5,6,7 --master_port 29501 tinyllava/train/train.py \
+    --deepspeed ./scripts/zero3.json \
+    --data_path  $DATA_PATH \
+    --image_folder $IMAGE_PATH \
+    --is_multimodal True \
+    --conv_version $CONV_VERSION \
+    --model_name_or_path $LLM_VERSION \
+    --vision_tower $VT_VERSION \
+    --vision_tower2 "$VT_VERSION2" \
+    --connector_type $CN_VERSION \
+    --mm_vision_select_layer -2 \
+    --image_aspect_ratio square \
+    --attn_implementation flash_attention_2 \
+    --bf16 True \
+    --training_recipe $TRAIN_RECIPE \
+    --tune_type_llm full \
+    --tune_type_vision_tower frozen\
+    --tune_vision_tower_from_layer 0 \
+    --tune_type_connector full \
+    --group_by_modality_length True \
+    --pretrained_model_path /mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-pretrain \
+    --output_dir /mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-finetune \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 4 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 4 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 50000 \
+    --save_total_limit 1 \
+    --learning_rate 2e-5 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 False \
+    --model_max_length $MODEL_MAX_LENGTH \
+    --gradient_checkpointing True \
+    --dataloader_num_workers 8 \
+    --lazy_preprocess True \
+    --report_to tensorboard \
+    --tokenizer_use_fast False \
+    --run_name tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-finetune

TinyLLaVA_Factory/scripts/train/qwen2/pretrain_qwen2.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+if [ $# -ne 9 ]; then
+    echo "Usage: $0 <DATA_PATH> <IMAGE_PATH> <LLM_VERSION> <VT_VERSION> <VT_VERSION2> <CN_VERSION> <VERSION> <TRAIN_RECIPE> <MODEL_MAX_LENGTH>"
+    exit 1
+fi
+
+# Assign the arguments to variables
+DATA_PATH="$1"
+IMAGE_PATH="$2"
+LLM_VERSION="$3"
+VT_VERSION="$4"
+VT_VERSION2="$5"
+CN_VERSION="$6"
+VERSION="$7"
+TRAIN_RECIPE="$8"
+MODEL_MAX_LENGTH="$9"
+
+VT_VARIANT="${VT_VERSION#*/}"
+LLM_VARIANT="${LLM_VERSION#*/}"
+
+deepspeed --include localhost:0,1,2,3,4,5,6,7 --master_port 29501 tinyllava/train/train.py \
+    --deepspeed ./scripts/zero3.json \
+    --data_path  $DATA_PATH\
+    --image_folder $IMAGE_PATH \
+    --is_multimodal True \
+    --conv_version pretrain \
+    --model_name_or_path $LLM_VERSION \
+    --vision_tower $VT_VERSION \
+    --vision_tower2 "$VT_VERSION2" \
+    --connector_type $CN_VERSION \
+    --mm_vision_select_layer -2 \
+    --image_aspect_ratio square \
+    --attn_implementation flash_attention_2 \
+    --bf16 True \
+    --training_recipe $TRAIN_RECIPE \
+    --tune_type_llm frozen \
+    --tune_type_vision_tower frozen \
+    --tune_vision_tower_from_layer 0 \
+    --tune_type_connector full \
+    --output_dir /mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-pretrain \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 16 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 2 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 24000 \
+    --save_total_limit 1 \
+    --learning_rate 1e-3 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 False \
+    --model_max_length $MODEL_MAX_LENGTH \
+    --gradient_checkpointing True \
+    --dataloader_num_workers 8 \
+    --lazy_preprocess True \
+    --report_to tensorboard \
+    --tokenizer_use_fast False \
+    --run_name tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-pretrain

TinyLLaVA_Factory/scripts/train/qwen2/readme.md

@@ -0,0 +1 @@
+These codes work for Qwen/Qwen2-0.5B and Qwen/Qwen2-0.5B-Instruct. However there is bug causing from deepspeed when you use other Qwen2 models like qwen2-1.5B. If you want to use tinyllava to train other qwen2 models, please feel free to contact our team.

TinyLLaVA_Factory/scripts/train/qwen2/train_qwen2_base.sh

@@ -0,0 +1,17 @@
+DATA_PATH=/home/ai/data/llava/dataset/text_files/blip_laion_cc_sbu_558k.json #pretrain annotation file path
+FINETUNE_DATA_PATH=/home/ai/data/llava/dataset/text_files/llava_v1_5_mix665k.json #finetune annotation file path
+IMAGE_PATH=/home/ai/data/llava/dataset/llava/llava_pretrain/images #pretrain image dir
+FINETUNE_IMAGE_PATH=/home/ai/data/llava/dataset #finetune image dir
+
+LLM_VERSION=Qwen/Qwen2-0.5B # llm path in huggingface
+VT_VERSION=google/siglip-so400m-patch14-384 #vision tower path in huggingface
+VT_VERSION2="" #if you are not using mof vision tower, keep it empty
+CN_VERSION=mlp2x_gelu #connector type, other options are: qformer, resampler, etc
+CONV_VERSION=qwen2_base #chat template, other options are: phi, llama, gemmma, etc
+VERSION=qwen2-0_5b_base #experiment name for recording different runnings
+TRAIN_RECIPE=common #training recipes, other options are: lora, qlora
+MODEL_MAX_LENGTH=2048 #max model length for llm
+
+
+bash scripts/train/qwen2/pretrain_qwen2.sh "$DATA_PATH" "$IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$VERSION" "$TRAIN_RECIPE" "$MODEL_MAX_LENGTH"
+bash scripts/train/qwen2/finetune_qwen2.sh "$FINETUNE_DATA_PATH" "$FINETUNE_IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$CONV_VERSION" "$VERSION" "$TRAIN_RECIPE" "$MODEL_MAX_LENGTH"

TinyLLaVA_Factory/scripts/train/qwen2/train_qwen2_instruct.sh

@@ -0,0 +1,17 @@
+DATA_PATH=/home/ai/data/llava/dataset/text_files/blip_laion_cc_sbu_558k.json #pretrain annotation file path
+FINETUNE_DATA_PATH=/home/ai/data/llava/dataset/text_files/llava_v1_5_mix665k.json #finetune annotation file path
+IMAGE_PATH=/home/ai/data/llava/dataset/llava/llava_pretrain/images #pretrain image dir
+FINETUNE_IMAGE_PATH=/home/ai/data/llava/dataset #finetune image dir
+
+LLM_VERSION=Qwen/Qwen2-0.5B-Instruct # llm path in huggingface
+VT_VERSION=google/siglip-so400m-patch14-384 #vision tower path in huggingface
+VT_VERSION2="" #if you are not using mof vision tower, keep it empty
+CN_VERSION=mlp2x_gelu #connector type, other options are: qformer, resampler, etc
+CONV_VERSION=qwen2_instruct #chat template, other options are: phi, llama, gemmma, etc
+VERSION=qwen2-0_5b_instruct #experiment name for recording different runnings
+TRAIN_RECIPE=common #training recipes, other options are: lora, qlora
+MODEL_MAX_LENGTH=2048 #max model length for llm
+
+
+bash scripts/train/qwen2/pretrain_qwen2.sh "$DATA_PATH" "$IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$VERSION" "$TRAIN_RECIPE" "$MODEL_MAX_LENGTH"
+bash scripts/train/qwen2/finetune_qwen2.sh "$FINETUNE_DATA_PATH" "$FINETUNE_IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$CONV_VERSION" "$VERSION" "$TRAIN_RECIPE" "$MODEL_MAX_LENGTH"

TinyLLaVA_Factory/scripts/train/share/finetune_share.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+if [ $# -ne 10 ]; then
+    echo "Usage: $0 <DATA_PATH> <IMAGE_PATH> <LLM_VERSION> <VT_VERSION> <VT_VERSION2> <CN_VERSION> <CONV_VERSION> <VERSION> <TRAIN_RECIPE> <MODEL_MAX_LENGTH>"
+    exit 1
+fi
+
+# Assign the arguments to variables
+DATA_PATH="$1"
+IMAGE_PATH="$2"
+LLM_VERSION="$3"
+VT_VERSION="$4"
+VT_VERSION2="$5"
+CN_VERSION="$6"
+CONV_VERSION="$7"
+VERSION="$8"
+TRAIN_RECIPE="$9"
+MODEL_MAX_LENGTH="${10}"
+
+VT_VARIANT="${VT_VERSION#*/}"
+LLM_VARIANT="${LLM_VERSION#*/}"
+
+deepspeed --include localhost:4,5,6,7 --master_port 29501 tinyllava/train/train.py \
+    --deepspeed ./scripts/zero3.json \
+    --data_path  $DATA_PATH \
+    --image_folder $IMAGE_PATH \
+    --is_multimodal True \
+    --conv_version $CONV_VERSION \
+    --model_name_or_path $LLM_VERSION \
+    --vision_tower $VT_VERSION \
+    --vision_tower2 "$VT_VERSION2" \
+    --connector_type $CN_VERSION \
+    --mm_vision_select_layer -2 \
+    --image_aspect_ratio square \
+    --attn_implementation flash_attention_2 \
+    --fp16 True \
+    --training_recipe $TRAIN_RECIPE \
+    --tune_type_llm full \
+    --tune_type_vision_tower frozen \
+    --tune_vision_tower_from_layer 0 \
+    --tune_type_connector full \
+    --group_by_modality_length True \
+    --pretrained_model_path /mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-share-pretrain \
+    --output_dir /mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-share-finetune \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 8 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 4 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 50000 \
+    --save_total_limit 1 \
+    --learning_rate 2e-5 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 False \
+    --model_max_length $MODEL_MAX_LENGTH \
+    --gradient_checkpointing True \
+    --dataloader_num_workers 8 \
+    --lazy_preprocess True \
+    --report_to tensorboard \
+    --tokenizer_use_fast False \
+    --run_name tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-share-finetune

TinyLLaVA_Factory/scripts/train/share/pretrain_share.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+if [ $# -ne 9 ]; then
+    echo "Usage: $0 <DATA_PATH> <IMAGE_PATH> <LLM_VERSION> <VT_VERSION> <VT_VERSION2> <CN_VERSION> <VERSION> <TRAIN_RECIPE> <MODEL_MAX_LENGTH>"
+    exit 1
+fi
+
+# Assign the arguments to variables
+DATA_PATH="$1"
+IMAGE_PATH="$2"
+LLM_VERSION="$3"
+VT_VERSION="$4"
+VT_VERSION2="$5"
+CN_VERSION="$6"
+VERSION="$7"
+TRAIN_RECIPE="$8"
+MODEL_MAX_LENGTH="$9"
+
+VT_VARIANT="${VT_VERSION#*/}"
+LLM_VARIANT="${LLM_VERSION#*/}"
+
+deepspeed --include localhost:4,5,6,7 --master_port 29501 tinyllava/train/train.py \
+    --deepspeed ./scripts/zero3.json \
+    --data_path  $DATA_PATH\
+    --image_folder $IMAGE_PATH \
+    --is_multimodal True \
+    --conv_version pretrain \
+    --model_name_or_path $LLM_VERSION \
+    --vision_tower $VT_VERSION \
+    --vision_tower2 "$VT_VERSION2" \
+    --connector_type $CN_VERSION \
+    --mm_vision_select_layer -2 \
+    --image_aspect_ratio square \
+    --attn_implementation flash_attention_2 \
+    --fp16 True \
+    --training_recipe $TRAIN_RECIPE \
+    --tune_type_llm full \
+    --tune_type_vision_tower frozen \
+    --tune_vision_tower_from_layer 0 \
+    --tune_type_connector full \
+    --pretrained_model_path /mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-pretrain \
+    --output_dir /mnt/data/sata/yinghu/checkpoints/llava_factory/tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-share-pretrain \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 8 \
+    --per_device_eval_batch_size 4 \
+    --gradient_accumulation_steps 8 \
+    --evaluation_strategy "no" \
+    --save_strategy "steps" \
+    --save_steps 24000 \
+    --save_total_limit 1 \
+    --learning_rate 2e-5 \
+    --weight_decay 0. \
+    --warmup_ratio 0.03 \
+    --lr_scheduler_type "cosine" \
+    --logging_steps 1 \
+    --tf32 False \
+    --model_max_length $MODEL_MAX_LENGTH \
+    --gradient_checkpointing True \
+    --dataloader_num_workers 8 \
+    --lazy_preprocess True \
+    --report_to tensorboard \
+    --tokenizer_use_fast False \
+    --run_name tiny-llava-${LLM_VARIANT}-${VT_VARIANT}-${VERSION}-share-pretrain

TinyLLaVA_Factory/scripts/train/share/train_phi_share.sh

@@ -0,0 +1,21 @@
+DATA_PATH=/home/ai/data/llava/dataset/text_files/blip_laion_cc_sbu_558k.json
+SHARE_PRETRAIN_DATA_PATH=/mnt/data/sata/ssd/dataset/text_files/really_cleaned_share-captioner_coco_lcs_sam_1246k_1107.json
+SHARE_FINETUNE_DATA_PATH=/mnt/data/sata/ssd/dataset/text_files/cleaned_sharegpt4v_mix665k_cap23k_coco-ap9k_lcs3k_sam9k_div2k.json
+IMAGE_PATH=/home/ai/data/llava/dataset/llava/llava_pretrain/images
+SHARE_PRETRAIN_IMAGE_PATH=/home/ai/data/llava/dataset
+SHARE_FINETUNE_IMAGE_PATH=/home/ai/data/llava/dataset
+
+LLM_VERSION=microsoft/phi-2
+VT_VERSION=google/siglip-so400m-patch14-384
+VT_VERSION2=""
+CN_VERSION=mlp2x_gelu
+CONV_VERSION=phi
+VERSION=share
+TRAIN_RECIPE=common
+MODEL_MAX_LENGTH=3072
+
+
+
+bash scripts/train/pretrain.sh "$DATA_PATH" "$IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$VERSION" "$TRAIN_RECIPE" "$MODEL_MAX_LENGTH"
+bash scripts/train/share/pretrain_share.sh "$SHARE_PRETRAIN_DATA_PATH" "$SHARE_PRETRAIN_IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$VERSION" "$TRAIN_RECIPE" "$MODEL_MAX_LENGTH" 
+bash scripts/train/share/finetune_share.sh "$SHARE_FINETUNE_DATA_PATH" "$SHARE_FINETUNE_IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$CONV_VERSION" "$VERSION" "$TRAIN_RECIPE" "$MODEL_MAX_LENGTH"

TinyLLaVA_Factory/scripts/train/train_mof.sh

@@ -0,0 +1,17 @@
+DATA_PATH=/home/ai/data/llava/dataset/text_files/blip_laion_cc_sbu_558k.json
+FINETUNE_DATA_PATH=/home/ai/data/llava/dataset/text_files/llava_v1_5_mix665k.json
+IMAGE_PATH=/home/ai/data/llava/dataset/llava/llava_pretrain/images
+FINETUNE_IMAGE_PATH=/home/ai/data/llava/dataset
+
+LLM_VERSION=TinyLlama/TinyLlama-1.1B-Chat-v1.0
+VT_VERSION=mof:openai/clip-vit-large-patch14
+VT_VERSION2=mof:facebook/dinov2-large
+CN_VERSION=mof_mlp
+CONV_VERSION=llama
+VERSION=llama-mof-base
+TRAIN_RECIPE=common
+MODEL_MAX_LENGTH=2048
+
+
+bash scripts/train/pretrain.sh "$DATA_PATH" "$IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$VERSION" "$TRAIN_RECIPE" "$MODEL_MAX_LENGTH"
+bash scripts/train/finetune.sh "$FINETUNE_DATA_PATH" "$FINETUNE_IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$CONV_VERSION" "$VERSION" "$TRAIN_RECIPE" "$MODEL_MAX_LENGTH"

TinyLLaVA_Factory/scripts/train/train_phi.sh

@@ -0,0 +1,17 @@
+DATA_PATH=/home/ai/data/llava/dataset/text_files/blip_laion_cc_sbu_558k.json #pretrain annotation file path
+FINETUNE_DATA_PATH=/home/ai/data/llava/dataset/text_files/llava_v1_5_mix665k.json #finetune annotation file path
+IMAGE_PATH=/home/ai/data/llava/dataset/llava/llava_pretrain/images #pretrain image dir
+FINETUNE_IMAGE_PATH=/home/ai/data/llava/dataset #finetune image dir
+
+LLM_VERSION=microsoft/phi-2 # llm path in huggingface
+VT_VERSION=google/siglip-so400m-patch14-384 #vision tower path in huggingface
+VT_VERSION2="" #if you are not using mof vision tower, keep it empty
+CN_VERSION=mlp2x_gelu #connector type, other options are: qformer, resampler, etc
+CONV_VERSION=phi #chat template, other options are: phi, llama, gemmma, etc
+VERSION=base #experiment name for recording different runnings
+TRAIN_RECIPE=common #training recipes, other options are: lora, qlora
+MODEL_MAX_LENGTH=3072 #max model length for llm
+
+
+bash scripts/train/pretrain.sh "$DATA_PATH" "$IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$VERSION" "$TRAIN_RECIPE" "$MODEL_MAX_LENGTH"
+bash scripts/train/finetune.sh "$FINETUNE_DATA_PATH" "$FINETUNE_IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$CONV_VERSION" "$VERSION" "$TRAIN_RECIPE" "$MODEL_MAX_LENGTH"

TinyLLaVA_Factory/scripts/train/train_stablelm.sh

@@ -0,0 +1,17 @@
+DATA_PATH=/home/ai/data/llava/dataset/text_files/blip_laion_cc_sbu_558k.json #pretrain annotation file path
+FINETUNE_DATA_PATH=/home/ai/data/llava/dataset/text_files/llava_v1_5_mix665k.json #finetune annotation file path
+IMAGE_PATH=/home/ai/data/llava/dataset/llava/llava_pretrain/images #pretrain image dir
+FINETUNE_IMAGE_PATH=/home/ai/data/llava/dataset #finetune image dir
+
+LLM_VERSION=stabilityai/stablelm-2-zephyr-1_6b # llm path in huggingface
+VT_VERSION=openai/clip-vit-large-patch14-336 #vision tower path in huggingface
+VT_VERSION2="" #if you are not using mof vision tower, keep it empty
+CN_VERSION=mlp2x_gelu #connector type, other options are: qformer, resampler, etc
+CONV_VERSION=phi #chat template for stablelm is the same as that for phi
+VERSION=base #experiment name for recording different runnings
+TRAIN_RECIPE=common #training recipes, other options are: lora, qlora
+MODEL_MAX_LENGTH=2048 #max model length for llm
+
+
+bash scripts/train/pretrain.sh "$DATA_PATH" "$IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$VERSION" "$TRAIN_RECIPE" "$MODEL_MAX_LENGTH"
+bash scripts/train/finetune.sh "$FINETUNE_DATA_PATH" "$FINETUNE_IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$CONV_VERSION" "$VERSION" "$TRAIN_RECIPE" "$MODEL_MAX_LENGTH"

TinyLLaVA_Factory/scripts/train/train_tinyllama.sh

@@ -0,0 +1,17 @@
+DATA_PATH=/home/ai/data/llava/dataset/text_files/blip_laion_cc_sbu_558k.json #pretrain annotation file path
+FINETUNE_DATA_PATH=/home/ai/data/llava/dataset/text_files/llava_v1_5_mix665k.json #finetune annotation file path
+IMAGE_PATH=/home/ai/data/llava/dataset/llava/llava_pretrain/images #pretrain image dir
+FINETUNE_IMAGE_PATH=/home/ai/data/llava/dataset #finetune image dir
+
+LLM_VERSION=TinyLlama/TinyLlama-1.1B-Chat-v1.0 # llm path in huggingface
+VT_VERSION=google/siglip-so400m-patch14-384 #vision tower path in huggingface
+VT_VERSION2="" #if you are not using mof vision tower, keep it empty
+CN_VERSION=mlp2x_gelu #connector type, other options are: qformer, resampler, etc
+CONV_VERSION=llama #chat template, other options are: phi, llama, gemmma, etc
+VERSION=base #experiment name for recording different runnings
+TRAIN_RECIPE=common #training recipes, other options are: lora, qlora
+MODEL_MAX_LENGTH=2048 #max model length for llm
+
+
+bash scripts/train/pretrain.sh "$DATA_PATH" "$IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$VERSION" "$TRAIN_RECIPE" "$MODEL_MAX_LENGTH"
+bash scripts/train/finetune.sh "$FINETUNE_DATA_PATH" "$FINETUNE_IMAGE_PATH" "$LLM_VERSION" "$VT_VERSION" "$VT_VERSION2" "$CN_VERSION" "$CONV_VERSION" "$VERSION" "$TRAIN_RECIPE" "$MODEL_MAX_LENGTH"

TinyLLaVA_Factory/scripts/zero2.json

@@ -0,0 +1,23 @@
+{
+    "fp16": {
+        "enabled": "auto",
+        "loss_scale": 0,
+        "loss_scale_window": 1000,
+        "initial_scale_power": 16,
+        "hysteresis": 2,
+        "min_loss_scale": 1
+    },
+    "bf16": {
+        "enabled": "auto"
+    },
+    "train_micro_batch_size_per_gpu": "auto",
+    "train_batch_size": "auto",
+    "gradient_accumulation_steps": "auto",
+    "zero_optimization": {
+        "stage": 2,
+        "overlap_comm": true,
+        "contiguous_gradients": true,
+        "sub_group_size": 1e9,
+        "reduce_bucket_size": "auto"
+    }
+}

TinyLLaVA_Factory/scripts/zero3.json

@@ -0,0 +1,28 @@
+{
+    "fp16": {
+        "enabled": "auto",
+        "loss_scale": 0,
+        "loss_scale_window": 1000,
+        "initial_scale_power": 16,
+        "hysteresis": 2,
+        "min_loss_scale": 1
+    },
+    "bf16": {
+        "enabled": "auto"
+    },
+    "train_micro_batch_size_per_gpu": "auto",
+    "train_batch_size": "auto",
+    "gradient_accumulation_steps": "auto",
+    "zero_optimization": {
+        "stage": 3,
+        "overlap_comm": true,
+        "contiguous_gradients": true,
+        "sub_group_size": 1e9,
+        "reduce_bucket_size": "auto",
+        "stage3_prefetch_bucket_size": "auto",
+        "stage3_param_persistence_threshold": "auto",
+        "stage3_max_live_parameters": 1e9,
+        "stage3_max_reuse_distance": 1e9,
+        "stage3_gather_16bit_weights_on_model_save": true
+    }
+}

TinyLLaVA_Factory/tinyllava/data/__init__.py

@@ -0,0 +1,4 @@
+from .template import *
+from .image_preprocess import *
+from .text_preprocess import *
+from .dataset import *

TinyLLaVA_Factory/tinyllava/data/dataset.py

@@ -0,0 +1,128 @@
+import copy
+from dataclasses import dataclass
+import json
+from typing import Dict,  Sequence, TYPE_CHECKING
+from PIL import Image, ImageFile
+import os
+
+from .text_preprocess import TextPreprocess
+from .image_preprocess import ImagePreprocess
+from ..utils.arguments import DataArguments
+from ..utils.constants import *
+
+
+import transformers
+import torch
+from torch.utils.data import Dataset
+
+
+
+ImageFile.LOAD_TRUNCATED_IMAGES = True
+
+class LazySupervisedDataset(Dataset):
+    """Dataset for supervised fine-tuning."""
+
+    def __init__(self, data_path: str,
+                 tokenizer: transformers.PreTrainedTokenizer,
+                 data_args: DataArguments):
+        super(LazySupervisedDataset, self).__init__()
+        list_data_dict = json.load(open(data_path, "r"))
+
+        self.tokenizer = tokenizer
+        self.list_data_dict = list_data_dict
+        self.data_args = data_args
+        self.text_preprocess = TextPreprocess(tokenizer, data_args.conv_version)
+        self.image_preprocess = ImagePreprocess(data_args.image_processor, data_args)
+
+    def __len__(self):
+        return len(self.list_data_dict)
+
+    @property
+    def lengths(self):
+        length_list = []
+        for sample in self.list_data_dict:
+            img_tokens = 128 if 'image' in sample else 0
+            length_list.append(sum(len(conv['value'].split()) for conv in sample['conversations']) + img_tokens)
+        return length_list
+
+    @property
+    def modality_lengths(self):
+        length_list = []
+        for sample in self.list_data_dict:
+            cur_len = sum(len(conv['value'].split()) for conv in sample['conversations'])
+            cur_len = cur_len if 'image' in sample else -cur_len
+            length_list.append(cur_len)
+        return length_list
+
+    def __getitem__(self, i) -> Dict[str, torch.Tensor]:
+        sources = self.list_data_dict[i]
+        data_dict = self.text_preprocess(copy.deepcopy(sources["conversations"]))
+        if 'image' in sources:
+            image_file = self.list_data_dict[i]['image']
+            image_folder = self.data_args.image_folder
+            image = Image.open(os.path.join(image_folder, image_file)).convert('RGB')
+            image = self.image_preprocess(image)
+            data_dict['image'] = image
+        elif self.data_args.is_multimodal:
+            # image does not exist in the data, but the model is multimodal
+            # print(f'{i}:{sources}')
+            crop_size = getattr(self.data_args.image_processor, 'crop_size', getattr(self.data_args.image_processor, 'size'))
+            data_dict['image'] = torch.zeros(3, crop_size['height'], crop_size['width'])
+        return data_dict
+
+
+@dataclass
+class DataCollatorForSupervisedDataset(object):
+    """Collate examples for supervised fine-tuning."""
+
+    tokenizer: transformers.PreTrainedTokenizer
+
+    def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
+        input_ids, labels = tuple([instance[key] for instance in instances]
+                                  for key in ("input_ids", "labels"))
+        if self.tokenizer.pad_token_id == self.tokenizer.eos_token_id:
+            for input_id in input_ids:
+                input_id[input_id == self.tokenizer.eos_token_id] = -300
+        input_ids = torch.nn.utils.rnn.pad_sequence(
+            input_ids,
+            batch_first=True,
+            padding_value=self.tokenizer.pad_token_id)
+        labels = torch.nn.utils.rnn.pad_sequence(labels,
+                                                 batch_first=True,
+                                                 padding_value=IGNORE_INDEX)
+        input_ids = input_ids[:, :self.tokenizer.model_max_length]
+        attention_mask = input_ids.ne(self.tokenizer.pad_token_id)
+        labels = labels[:, :self.tokenizer.model_max_length]
+        # FIXME: This is a hack for handling phi and stablelm, as they have the same eos, pad and unk. We want the model
+        # FIXME: to predict the eos in the input ids, but we also use the id of eos to pad sequence, so we use a temp
+        # FIXME: eos id first, and convert them back.
+        if self.tokenizer.pad_token_id == self.tokenizer.eos_token_id:
+            for input_id in input_ids:
+                input_id[input_id == -300] = self.tokenizer.eos_token_id
+
+        batch = dict(
+            input_ids=input_ids,
+            labels=labels,
+            attention_mask=attention_mask,
+        )
+
+        if 'image' in instances[0]:
+            images = [instance['image'] for instance in instances]
+            if all(x is not None and x.shape == images[0].shape for x in images):
+                batch['images'] = torch.stack(images)
+            else:
+                batch['images'] = images
+
+        return batch
+
+
+def make_supervised_data_module(tokenizer: transformers.PreTrainedTokenizer,
+                                data_args) -> Dict:
+    """Make dataset and collator for supervised fine-tuning."""
+    train_dataset = LazySupervisedDataset(tokenizer=tokenizer,
+                                          data_path=data_args.data_path,
+                                          data_args=data_args)
+    data_collator = DataCollatorForSupervisedDataset(tokenizer=tokenizer)
+    return dict(train_dataset=train_dataset,
+                eval_dataset=None,
+                data_collator=data_collator)

TinyLLaVA_Factory/tinyllava/data/image_preprocess.py

@@ -0,0 +1,70 @@
+import os
+
+from PIL import Image, ImageFile
+import torch
+import ast
+
+from ..utils.data_utils import *
+
+ImageFile.LOAD_TRUNCATED_IMAGES = True
+
+# 可能imagepreprocess需要继承一个huggingface的图像处理类？提供from_pretrained方法
+
+class ImagePreprocess:
+    def __init__(self, image_processor, data_args={}):
+        self.image_aspect_ratio = getattr(data_args, 'image_aspect_ratio', None)
+        self.image_processor = image_processor
+        self.image_grid_pinpoints = getattr(data_args, 'image_grid_pinpoints', None)
+    
+    def __call__(self, image):
+        if self.image_aspect_ratio == 'pad':
+            image = self.expand2square(image, tuple(int(x * 255) for x in self.image_processor.image_mean))
+        elif self.image_aspect_ratio == "anyres":
+            image = self.process_anyres_image(image, self.image_processor, self.image_grid_pinpoints)
+            return image
+        image = self.image_processor(image, return_tensors='pt')['pixel_values'][0]
+        return image
+
+    @classmethod
+    def expand2square(cls, pil_img, background_color):
+        width, height = pil_img.size
+        if width == height:
+            return pil_img
+        elif width > height:
+            result = Image.new(pil_img.mode, (width, width), background_color)
+            result.paste(pil_img, (0, (width - height) // 2))
+            return result
+        else:
+            result = Image.new(pil_img.mode, (height, height), background_color)
+            result.paste(pil_img, ((height - width) // 2, 0))
+            return result
+    
+    @classmethod
+    def process_anyres_image(cls, image, processor, grid_pinpoints):
+        """
+        Process an image with variable resolutions.
+
+        Args:
+            image (PIL.Image.Image): The input image to be processed.
+            processor: The image processor object.
+            grid_pinpoints (str): A string representation of a list of possible resolutions.
+
+        Returns:
+            torch.Tensor: A tensor containing the processed image patches.
+        """
+        if type(grid_pinpoints) is list:
+            possible_resolutions = grid_pinpoints
+        else:
+            possible_resolutions = ast.literal_eval(grid_pinpoints)
+        best_resolution = select_best_resolution(image.size, possible_resolutions)
+        image_padded = resize_and_pad_image(image, best_resolution)
+
+        patches = divide_to_patches(image_padded, processor.crop_size['height'])
+
+        image_original_resize = image.resize((processor.size['shortest_edge'], processor.size['shortest_edge']))
+
+        image_patches = [image_original_resize] + patches
+        image_patches = [processor(image_patch, return_tensors='pt')['pixel_values'][0]
+                        for image_patch in image_patches]
+        return torch.stack(image_patches, dim=0)
+    

TinyLLaVA_Factory/tinyllava/data/template/__init__.py

@@ -0,0 +1,29 @@
+import os
+from typing import Dict
+
+from .base import *
+from ...utils import import_modules
+
+
+TEMPlATE_FACTORY: Dict[str, Template] = {}
+
+def TemplateFactory(version):
+    template = TEMPlATE_FACTORY.get(version, None)
+    assert template, f"{version} is not implmentation"
+    return template
+
+
+def register_template(name):
+    def register_template_cls(cls):
+        if name in TEMPlATE_FACTORY:
+            return TEMPlATE_FACTORY[name]
+
+        TEMPlATE_FACTORY[name] = cls
+        return cls
+
+    return register_template_cls
+
+
+# automatically import any Python files in the models/ directory
+models_dir = os.path.dirname(__file__)
+import_modules(models_dir, "tinyllava.data.template")