[refactor] prepare packaging moving all the modules under 'lisa_on_cuda' (renamed from 'model')

.idea/lisa-on-gpu.iml

@@ -4,7 +4,7 @@
     <content url="file://$MODULE_DIR$">
       <excludeFolder url="file://$MODULE_DIR$/venv" />
     </content>
-    <orderEntry type="inheritedJdk" />
+    <orderEntry type="jdk" jdkName="Python 3.10 (lisa_on_cuda)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
   <component name="PyDocumentationSettings">

.idea/misc.xml

@@ -3,5 +3,5 @@
   <component name="Black">
     <option name="sdkName" value="Python 3.10 (lisa-on-gpu)" />
   </component>
-  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (lisa-on-gpu)" project-jdk-type="Python SDK" />
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (lisa_on_cuda)" project-jdk-type="Python SDK" />
 </project>

{app → lisa_on_cuda/app}/chat.py

@@ -7,11 +7,11 @@ import numpy as np
 import torch
 from transformers import AutoTokenizer, BitsAndBytesConfig, CLIPImageProcessor
 
-from model.LISA import LISAForCausalLM
-from model.llava import conversation as conversation_lib
-from model.llava.mm_utils import tokenizer_image_token
-from model.segment_anything.utils.transforms import ResizeLongestSide
-from utils import app_helpers, utils
+from lisa_on_cuda.LISA import LISAForCausalLM
+from lisa_on_cuda.llava import conversation as conversation_lib
+from lisa_on_cuda.llava.mm_utils import tokenizer_image_token
+from lisa_on_cuda.segment_anything.utils.transforms import ResizeLongestSide
+from ..utils import app_helpers, utils
 
 
 def main(args):

{app → lisa_on_cuda/app}/main.py

@@ -6,8 +6,8 @@ from fastapi import FastAPI
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 
-from app import routes
-from utils import app_helpers, session_logger
+from . import routes
+from ..utils import app_helpers, session_logger
 
 
 session_logger.change_logging(logging.DEBUG)

{app → lisa_on_cuda/app}/merge_lora_weights_and_save_hf_model.py

@@ -11,8 +11,8 @@ import transformers
 from peft import LoraConfig, get_peft_model
 from transformers import AutoTokenizer
 
-from model.LISA import LISAForCausalLM
-from utils.utils import DEFAULT_IM_END_TOKEN, DEFAULT_IM_START_TOKEN
+from lisa_on_cuda.LISA import LISAForCausalLM
+from ..utils.utils import DEFAULT_IM_END_TOKEN, DEFAULT_IM_START_TOKEN
 
 
 def parse_args(args):

{app → lisa_on_cuda/app}/routes.py

@@ -2,7 +2,7 @@ import json
 import logging
 from fastapi import APIRouter
 
-from utils import session_logger
+from ..utils import session_logger
 
 
 router = APIRouter()

{app → lisa_on_cuda/app}/train_ds.py

@@ -13,10 +13,10 @@ import transformers
 from peft import LoraConfig, get_peft_model
 from torch.utils.tensorboard import SummaryWriter
 
-from model.LISA import LISAForCausalLM
-from model.llava import conversation as conversation_lib
-from utils.dataset import HybridDataset, ValDataset, collate_fn
-from utils.utils import (DEFAULT_IM_END_TOKEN, DEFAULT_IM_START_TOKEN,
+from lisa_on_cuda.LISA import LISAForCausalLM
+from lisa_on_cuda.llava import conversation as conversation_lib
+from ..utils.dataset import HybridDataset, ValDataset, collate_fn
+from ..utils.utils import (DEFAULT_IM_END_TOKEN, DEFAULT_IM_START_TOKEN,
                          AverageMeter, ProgressMeter, Summary, dict_to_cuda,
                          intersectionAndUnionGPU)
 

{model → lisa_on_cuda}/llava/model/apply_delta.py

@@ -5,10 +5,11 @@ python3 -m fastchat.model.apply_delta --base ~/model_weights/llama-7b --target ~
 import argparse
 
 import torch
-from llava import LlavaLlamaForCausalLM
 from tqdm import tqdm
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
+from .language_model.llava_llama import LlavaLlamaForCausalLM
+
 
 def apply_delta(base_model_path, target_model_path, delta_path):
     print("Loading base model")

{model → lisa_on_cuda}/llava/model/builder.py

@@ -17,11 +17,11 @@ import os
 import shutil
 
 import torch
-from llava.constants import (DEFAULT_IM_END_TOKEN, DEFAULT_IM_START_TOKEN,
-                             DEFAULT_IMAGE_PATCH_TOKEN)
-from llava.model import *
-from transformers import (AutoConfig, AutoModelForCausalLM, AutoTokenizer,
-                          BitsAndBytesConfig)
+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+
+from .language_model.llava_llama import LlavaLlamaForCausalLM
+from .language_model.llava_mpt import LlavaMPTForCausalLM
+from ..constants import DEFAULT_IM_END_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IMAGE_PATCH_TOKEN
 
 
 def load_pretrained_model(

{model → lisa_on_cuda}/llava/model/consolidate.py

@@ -5,8 +5,7 @@ python3 -m llava.model.consolidate --src ~/model_weights/llava-7b --dst ~/model_
 import argparse
 
 import torch
-from llava.model import *
-from llava.model.utils import auto_upgrade
+from .utils import auto_upgrade
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
 

{model → lisa_on_cuda}/llava/model/llava_arch.py

@@ -18,10 +18,7 @@ from abc import ABC, abstractmethod
 import torch
 import torch.nn as nn
 
-# from llava.constants import IGNORE_INDEX, IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
-from utils.utils import (DEFAULT_IM_END_TOKEN, DEFAULT_IM_START_TOKEN,
-                         DEFAULT_IMAGE_PATCH_TOKEN, IGNORE_INDEX,
-                         IMAGE_TOKEN_INDEX)
+from lisa_on_cuda.utils.utils import IGNORE_INDEX, IMAGE_TOKEN_INDEX
 
 from .multimodal_encoder.builder import build_vision_tower
 

{model → lisa_on_cuda}/llava/model/make_delta.py

@@ -5,7 +5,7 @@ python3 -m llava.model.make_delta --base ~/model_weights/llama-7b --target ~/mod
 import argparse
 
 import torch
-from llava.model.utils import auto_upgrade
+from .utils import auto_upgrade
 from tqdm import tqdm
 from transformers import AutoModelForCausalLM, AutoTokenizer
 

{model → lisa_on_cuda}/llava/train/train_mem.py

@@ -3,12 +3,12 @@
 # Make it more memory efficient by monkey patching the LLaMA model with FlashAttn.
 
 # Need to call this before importing transformers.
-from llava.train.llama_flash_attn_monkey_patch import \
-    replace_llama_attn_with_flash_attn
+from .llama_flash_attn_monkey_patch import replace_llama_attn_with_flash_attn
 
 replace_llama_attn_with_flash_attn()
 
-from llava.train.train import train
+from .train import train
+
 
 if __name__ == "__main__":
     train()

{model → lisa_on_cuda}/llava/utils.py

@@ -5,7 +5,7 @@ import os
 import sys
 
 import requests
-from llava.constants import LOGDIR
+from .constants import LOGDIR
 
 server_error_msg = (
     "**NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.**"