init

iter_8721.pth/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2f5a84387c6351ccf3016f7558b040d98ddaf8b334326cfa6b264d55ba59129
+size 19676784

iter_8721.pth/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b779a7d3003162fefdb0361c75f340f6d0e2e73f50b6d0bec8e95c4e180e74c9
+size 19676912

iter_8721.pth/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67a083fe2a0f23b7307593f8be03f4865d8e198b884dfb14cf78cc0530ff6a7b
+size 19676784

iter_8721.pth/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e45c1d8bdac4984bc0c4b7aad3ebe14dc73145337ffd9c4bd7d4f82016c0d82b
+size 19676848

iter_8721.pth/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1ab247dc95e126f4b705a008e5e2d5a0e9e49588c38101a8d8a7844705e63e5
+size 13767724

pretrain.py

@@ -0,0 +1,202 @@
+SYSTEM = ''
+accumulative_counts = 4
+batch_size = 16
+betas = (
+    0.9,
+    0.999,
+)
+custom_hooks = [
+    dict(
+        tokenizer=dict(
+            padding_side='right',
+            pretrained_model_name_or_path='internlm/internlm2-chat-1_8b',
+            trust_remote_code=True,
+            type='transformers.AutoTokenizer.from_pretrained'),
+        type='xtuner.engine.hooks.DatasetInfoHook'),
+    dict(
+        evaluation_images='https://llava-vl.github.io/static/images/view.jpg',
+        evaluation_inputs=[
+            '请描述一下这张照片',
+            'Please describe this picture',
+        ],
+        every_n_iters=2000,
+        image_processor=dict(
+            pretrained_model_name_or_path='google/siglip-so400m-patch14-384',
+            trust_remote_code=True,
+            type='transformers.SiglipImageProcessor.from_pretrained'),
+        prompt_template='xtuner.utils.PROMPT_TEMPLATE.internlm2_chat',
+        system='',
+        tokenizer=dict(
+            padding_side='right',
+            pretrained_model_name_or_path='internlm/internlm2-chat-1_8b',
+            trust_remote_code=True,
+            type='transformers.AutoTokenizer.from_pretrained'),
+        type='xtuner.engine.hooks.EvaluateChatHook'),
+]
+data_path = './LLaVA-Pretrain/blip_laion_cc_sbu_558k.json'
+data_root = './'
+dataloader_num_workers = 16
+default_hooks = dict(
+    checkpoint=dict(
+        by_epoch=False,
+        interval=2000,
+        max_keep_ckpts=2,
+        type='mmengine.hooks.CheckpointHook'),
+    logger=dict(
+        interval=10,
+        log_metric_by_epoch=False,
+        type='mmengine.hooks.LoggerHook'),
+    param_scheduler=dict(type='mmengine.hooks.ParamSchedulerHook'),
+    sampler_seed=dict(type='mmengine.hooks.DistSamplerSeedHook'),
+    timer=dict(type='mmengine.hooks.IterTimerHook'))
+env_cfg = dict(
+    cudnn_benchmark=False,
+    dist_cfg=dict(backend='nccl'),
+    mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0))
+evaluation_freq = 2000
+evaluation_images = 'https://llava-vl.github.io/static/images/view.jpg'
+evaluation_inputs = [
+    '请描述一下这张照片',
+    'Please describe this picture',
+]
+image_folder = './LLaVA-Pretrain/images'
+image_processor = dict(
+    pretrained_model_name_or_path='google/siglip-so400m-patch14-384',
+    trust_remote_code=True,
+    type='transformers.SiglipImageProcessor.from_pretrained')
+launcher = 'pytorch'
+llava_dataset = dict(
+    data_path='./LLaVA-Pretrain/blip_laion_cc_sbu_558k.json',
+    dataset_map_fn='xtuner.dataset.map_fns.llava_map_fn',
+    image_folder='./LLaVA-Pretrain/images',
+    image_processor=dict(
+        pretrained_model_name_or_path='google/siglip-so400m-patch14-384',
+        trust_remote_code=True,
+        type='transformers.SiglipImageProcessor.from_pretrained'),
+    max_length=1472,
+    pad_image_to_square=False,
+    template_map_fn=dict(
+        template='xtuner.utils.PROMPT_TEMPLATE.internlm2_chat',
+        type='xtuner.dataset.map_fns.template_map_fn_factory'),
+    tokenizer=dict(
+        padding_side='right',
+        pretrained_model_name_or_path='internlm/internlm2-chat-1_8b',
+        trust_remote_code=True,
+        type='transformers.AutoTokenizer.from_pretrained'),
+    type='xtuner.dataset.LLaVADataset')
+llm_name_or_path = 'internlm/internlm2-chat-1_8b'
+load_from = None
+log_level = 'INFO'
+log_processor = dict(by_epoch=False)
+lr = 0.001
+max_epochs = 1
+max_length = 1472
+max_norm = 1
+model = dict(
+    freeze_llm=True,
+    freeze_visual_encoder=True,
+    llm=dict(
+        pretrained_model_name_or_path='internlm/internlm2-chat-1_8b',
+        quantization_config=dict(
+            bnb_4bit_compute_dtype='torch.float16',
+            bnb_4bit_quant_type='nf4',
+            bnb_4bit_use_double_quant=True,
+            llm_int8_has_fp16_weight=False,
+            llm_int8_threshold=6.0,
+            load_in_4bit=True,
+            load_in_8bit=False,
+            type='transformers.BitsAndBytesConfig'),
+        torch_dtype='torch.float16',
+        trust_remote_code=True,
+        type='transformers.AutoModelForCausalLM.from_pretrained'),
+    type='xtuner.model.LLaVAModel',
+    visual_encoder=dict(
+        pretrained_model_name_or_path='google/siglip-so400m-patch14-384',
+        type='transformers.SiglipVisionModel.from_pretrained'))
+optim_type = 'torch.optim.AdamW'
+optim_wrapper = dict(
+    optimizer=dict(
+        betas=(
+            0.9,
+            0.999,
+        ),
+        lr=0.001,
+        type='torch.optim.AdamW',
+        weight_decay=0),
+    type='DeepSpeedOptimWrapper')
+param_scheduler = [
+    dict(
+        begin=0,
+        by_epoch=True,
+        convert_to_iter_based=True,
+        end=0.03,
+        start_factor=1e-05,
+        type='mmengine.optim.LinearLR'),
+    dict(
+        begin=0.03,
+        by_epoch=True,
+        convert_to_iter_based=True,
+        end=1,
+        eta_min=0.0,
+        type='mmengine.optim.CosineAnnealingLR'),
+]
+prompt_template = 'xtuner.utils.PROMPT_TEMPLATE.internlm2_chat'
+randomness = dict(deterministic=False, seed=None)
+resume = False
+runner_type = 'FlexibleRunner'
+save_steps = 2000
+save_total_limit = 2
+strategy = dict(
+    config=dict(
+        bf16=dict(enabled=True),
+        fp16=dict(enabled=False, initial_scale_power=16),
+        gradient_accumulation_steps='auto',
+        gradient_clipping='auto',
+        train_micro_batch_size_per_gpu='auto',
+        zero_allow_untested_optimizer=True,
+        zero_force_ds_cpu_optimizer=False,
+        zero_optimization=dict(overlap_comm=True, stage=2)),
+    exclude_frozen_parameters=True,
+    gradient_accumulation_steps=4,
+    gradient_clipping=1,
+    train_micro_batch_size_per_gpu=16,
+    type='xtuner.engine.DeepSpeedStrategy')
+tokenizer = dict(
+    padding_side='right',
+    pretrained_model_name_or_path='internlm/internlm2-chat-1_8b',
+    trust_remote_code=True,
+    type='transformers.AutoTokenizer.from_pretrained')
+train_cfg = dict(max_epochs=1, type='xtuner.engine.runner.TrainLoop')
+train_dataloader = dict(
+    batch_size=16,
+    collate_fn=dict(type='xtuner.dataset.collate_fns.default_collate_fn'),
+    dataset=dict(
+        data_path='./LLaVA-Pretrain/blip_laion_cc_sbu_558k.json',
+        dataset_map_fn='xtuner.dataset.map_fns.llava_map_fn',
+        image_folder='./LLaVA-Pretrain/images',
+        image_processor=dict(
+            pretrained_model_name_or_path='google/siglip-so400m-patch14-384',
+            trust_remote_code=True,
+            type='transformers.SiglipImageProcessor.from_pretrained'),
+        max_length=1472,
+        pad_image_to_square=False,
+        template_map_fn=dict(
+            template='xtuner.utils.PROMPT_TEMPLATE.internlm2_chat',
+            type='xtuner.dataset.map_fns.template_map_fn_factory'),
+        tokenizer=dict(
+            padding_side='right',
+            pretrained_model_name_or_path='internlm/internlm2-chat-1_8b',
+            trust_remote_code=True,
+            type='transformers.AutoTokenizer.from_pretrained'),
+        type='xtuner.dataset.LLaVADataset'),
+    num_workers=16,
+    sampler=dict(shuffle=True, type='mmengine.dataset.DefaultSampler'))
+visual_encoder_name_or_path = 'google/siglip-so400m-patch14-384'
+visualizer = dict(
+    type='mmengine.visualization.Visualizer',
+    vis_backends=[
+        dict(type='mmengine.visualization.TensorboardVisBackend'),
+    ])
+warmup_ratio = 0.03
+weight_decay = 0
+work_dir = './work_dirs/pretrain'