diff --git "a/334457.err" "b/334457.err" new file mode 100644--- /dev/null +++ "b/334457.err" @@ -0,0 +1,8246 @@ + +CommandNotFoundError: Your shell has not been properly configured to use 'conda activate'. +To initialize your shell, run + + $ conda init + +Currently supported shells are: + - bash + - fish + - tcsh + - xonsh + - zsh + - powershell + +See 'conda init --help' for more information and options. + +IMPORTANT: You may need to close and restart your shell after running 'conda init'. + + +W0215 02:16:02.635000 139602503870272 torch/distributed/run.py:779] +W0215 02:16:02.635000 139602503870272 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.635000 139602503870272 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 02:16:02.635000 139602503870272 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 133614182537024 torch/distributed/run.py:779] +W0215 02:16:02.636000 133614182537024 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 133614182537024 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 02:16:02.636000 133614182537024 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 130114260338496 torch/distributed/run.py:779] +W0215 02:16:02.636000 130114260338496 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 130114260338496 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 02:16:02.636000 130114260338496 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 124609366361920 torch/distributed/run.py:779] +W0215 02:16:02.636000 124609366361920 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 124609366361920 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 02:16:02.636000 124609366361920 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 129987096590144 torch/distributed/run.py:779] +W0215 02:16:02.636000 129987096590144 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 129987096590144 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 02:16:02.636000 129987096590144 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 123775382849344 torch/distributed/run.py:779] +W0215 02:16:02.636000 123775382849344 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 123775382849344 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 02:16:02.636000 123775382849344 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 131818974000960 torch/distributed/run.py:779] +W0215 02:16:02.636000 131818974000960 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 131818974000960 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 02:16:02.636000 131818974000960 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 134719829468992 torch/distributed/run.py:779] +W0215 02:16:02.636000 134719829468992 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 134719829468992 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 02:16:02.636000 134719829468992 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 127727230662464 torch/distributed/run.py:779] +W0215 02:16:02.636000 127727230662464 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 127727230662464 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 02:16:02.636000 127727230662464 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 140083807930176 torch/distributed/run.py:779] +W0215 02:16:02.636000 140083807930176 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 140083807930176 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 02:16:02.636000 140083807930176 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 136148163000128 torch/distributed/run.py:779] +W0215 02:16:02.636000 136148163000128 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 136148163000128 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 02:16:02.636000 136148163000128 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 124877567706944 torch/distributed/run.py:779] +W0215 02:16:02.636000 124877567706944 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 124877567706944 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 02:16:02.636000 124877567706944 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 128235731347264 torch/distributed/run.py:779] +W0215 02:16:02.636000 128235731347264 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 128235731347264 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 02:16:02.636000 128235731347264 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 125992953558848 torch/distributed/run.py:779] +W0215 02:16:02.636000 125992953558848 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 125992953558848 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 02:16:02.636000 125992953558848 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 128930927953728 torch/distributed/run.py:779] +W0215 02:16:02.636000 128930927953728 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 128930927953728 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 02:16:02.636000 128930927953728 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 127343532558144 torch/distributed/run.py:779] +W0215 02:16:02.636000 127343532558144 torch/distributed/run.py:779] ***************************************** +W0215 02:16:02.636000 127343532558144 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0215 02:16:02.636000 127343532558144 torch/distributed/run.py:779] ***************************************** +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:211: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_fwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/xformers/ops/fmha/flash.py:344: FutureWarning: `torch.library.impl_abstract` was renamed to `torch.library.register_fake`. Please use that instead; we will remove `torch.library.impl_abstract` in a future version of PyTorch. + @torch.library.impl_abstract("xformers_flash::flash_bwd") +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + from llava.train.train import train +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from llava.train.train import trainfrom llava.train.train import trainfrom llava.train.train import train + + +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train +from llava.train.train import train + from llava.train.train import train +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from llava.train.train import train +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +from llava.train.train import train +from llava.train.train import train + Traceback (most recent call last): +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import trainfrom llava.train.train import train + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train +Traceback (most recent call last): + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from llava.train.train import train +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + from llava.train.train import train +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + from .model import LlavaLlamaForCausalLM +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import trainfrom llava.train.train import train + +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import trainfrom llava.train.train import train + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLM + from llava.train.train import trainfrom llava.train.train import train + + from llava.train.train import train + from llava.train.train import train + from llava.train.train import train + from llava.train.train import train +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +from .model import LlavaLlamaForCausalLM +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLM + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + +from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM +from llava.train.train import train File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +from llava.train.train import train File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.train.train import trainfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import trainfrom llava.train.train import train + + from llava.train.train import train +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM +Traceback (most recent call last): + from llava.train.train import train + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + + from .model import LlavaLlamaForCausalLM + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM + from llava.train.train import train + from .model import LlavaLlamaForCausalLM +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from llava.train.train import train + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +from llava.train.train import train + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import trainfrom llava.train.train import train + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +Traceback (most recent call last): + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.train.train import train +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLMfrom llava.train.train import train + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from llava.train.train import trainfrom llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from llava.train.train import train + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .model import LlavaLlamaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + +Traceback (most recent call last): + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from llava.train.train import trainfrom llava.train.train import train + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +Traceback (most recent call last): + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +from llava.train.train import train File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + + from .model import LlavaLlamaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + from .model import LlavaLlamaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + from .model import LlavaLlamaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from llava.train.train import train +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.train.train import trainfrom llava.train.train import trainfrom .model import LlavaLlamaForCausalLM + + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import trainfrom llava.train.train import train + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from llava.train.train import train + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .model import LlavaLlamaForCausalLM +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from llava.train.train import train + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .model import LlavaLlamaForCausalLMTraceback (most recent call last): + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from llava.train.train import trainfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigTraceback (most recent call last): + + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .model import LlavaLlamaForCausalLM + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from llava.train.train import trainfrom llava.train.train import train + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.train.train import train + from llava.train.train import train + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .model import LlavaLlamaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLMfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +Traceback (most recent call last): + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +Traceback (most recent call last): + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .model import LlavaLlamaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.train.train import trainfrom llava.train.train import trainfrom llava.train.train import train + + + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + + + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + from .model import LlavaLlamaForCausalLM + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +from .model import LlavaLlamaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + from .model import LlavaLlamaForCausalLM + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError : from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +: + cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError : from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminacannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportError + +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError ImportErrorfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina: : +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +ImportErrorImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +ImportError : from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + +ImportErrorImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportErrorfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + ImportErrorfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina: +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportError: ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError : from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportErrorfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina: + +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + ImportErrorfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina: + +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + ImportErrorfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +: from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminacannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) ImportError +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina : +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError +: ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import trainfrom llava.train.train import train + + from llava.train.train import train +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +from .model import LlavaLlamaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + ImportErrorfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina: +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +W0215 02:16:38.558000 133614182537024 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1186439 closing signal SIGTERM +W0215 02:16:38.558000 133614182537024 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1186440 closing signal SIGTERM +W0215 02:16:38.558000 133614182537024 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1186441 closing signal SIGTERM +W0215 02:16:38.558000 133614182537024 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1186443 closing signal SIGTERM +W0215 02:16:38.559000 133614182537024 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1186444 closing signal SIGTERM +W0215 02:16:38.559000 133614182537024 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1186445 closing signal SIGTERM +W0215 02:16:38.559000 133614182537024 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1186446 closing signal SIGTERM +W0215 02:16:38.560000 128930927953728 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1500456 closing signal SIGTERM +W0215 02:16:38.561000 128930927953728 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1500457 closing signal SIGTERM +W0215 02:16:38.562000 128930927953728 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1500459 closing signal SIGTERM +W0215 02:16:38.563000 128930927953728 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1500460 closing signal SIGTERM +W0215 02:16:38.563000 128930927953728 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1500461 closing signal SIGTERM +W0215 02:16:38.565000 128930927953728 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1500462 closing signal SIGTERM +W0215 02:16:38.565000 128930927953728 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1500463 closing signal SIGTERM +W0215 02:16:38.660000 136148163000128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 205123 closing signal SIGTERM +W0215 02:16:38.660000 136148163000128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 205124 closing signal SIGTERM +W0215 02:16:38.660000 136148163000128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 205125 closing signal SIGTERM +W0215 02:16:38.660000 136148163000128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 205126 closing signal SIGTERM +W0215 02:16:38.661000 136148163000128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 205129 closing signal SIGTERM +W0215 02:16:38.661000 136148163000128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 205130 closing signal SIGTERM +W0215 02:16:38.661000 130114260338496 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1194200 closing signal SIGTERM +W0215 02:16:38.662000 130114260338496 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1194201 closing signal SIGTERM +W0215 02:16:38.662000 130114260338496 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1194202 closing signal SIGTERM +W0215 02:16:38.662000 130114260338496 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1194203 closing signal SIGTERM +W0215 02:16:38.662000 130114260338496 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1194204 closing signal SIGTERM +W0215 02:16:38.662000 130114260338496 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1194207 closing signal SIGTERM +W0215 02:16:38.662000 129987096590144 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 810034 closing signal SIGTERM +W0215 02:16:38.663000 129987096590144 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 810035 closing signal SIGTERM +W0215 02:16:38.663000 129987096590144 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 810036 closing signal SIGTERM +W0215 02:16:38.664000 129987096590144 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 810040 closing signal SIGTERM +W0215 02:16:38.664000 129987096590144 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 810041 closing signal SIGTERM +W0215 02:16:38.762000 134719829468992 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 226745 closing signal SIGTERM +W0215 02:16:38.762000 134719829468992 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 226747 closing signal SIGTERM +W0215 02:16:38.762000 134719829468992 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 226748 closing signal SIGTERM +W0215 02:16:38.763000 134719829468992 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 226749 closing signal SIGTERM +W0215 02:16:38.763000 134719829468992 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 226750 closing signal SIGTERM +W0215 02:16:38.763000 134719829468992 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 226751 closing signal SIGTERM +W0215 02:16:38.778000 127343532558144 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2949104 closing signal SIGTERM +W0215 02:16:38.779000 127343532558144 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2949105 closing signal SIGTERM +W0215 02:16:38.779000 127343532558144 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2949106 closing signal SIGTERM +W0215 02:16:38.779000 127343532558144 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2949107 closing signal SIGTERM +W0215 02:16:38.779000 127343532558144 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2949108 closing signal SIGTERM +W0215 02:16:38.779000 127343532558144 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2949109 closing signal SIGTERM +W0215 02:16:38.779000 127343532558144 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2949110 closing signal SIGTERM +W0215 02:16:38.860000 131818974000960 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 821957 closing signal SIGTERM +W0215 02:16:38.861000 131818974000960 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 821958 closing signal SIGTERM +W0215 02:16:38.861000 131818974000960 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 821959 closing signal SIGTERM +W0215 02:16:38.861000 131818974000960 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 821960 closing signal SIGTERM +W0215 02:16:38.861000 131818974000960 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 821961 closing signal SIGTERM +W0215 02:16:38.861000 131818974000960 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 821962 closing signal SIGTERM +W0215 02:16:38.862000 131818974000960 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 821964 closing signal SIGTERM +W0215 02:16:38.961000 140083807930176 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 205435 closing signal SIGTERM +W0215 02:16:38.961000 140083807930176 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 205436 closing signal SIGTERM +W0215 02:16:38.961000 140083807930176 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 205437 closing signal SIGTERM +W0215 02:16:38.961000 140083807930176 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 205438 closing signal SIGTERM +W0215 02:16:38.961000 140083807930176 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 205439 closing signal SIGTERM +W0215 02:16:38.961000 140083807930176 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 205441 closing signal SIGTERM +W0215 02:16:38.961000 128235731347264 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 310640 closing signal SIGTERM +W0215 02:16:38.962000 128235731347264 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 310643 closing signal SIGTERM +W0215 02:16:38.962000 128235731347264 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 310644 closing signal SIGTERM +W0215 02:16:38.962000 128235731347264 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 310646 closing signal SIGTERM +W0215 02:16:39.062000 125992953558848 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 368407 closing signal SIGTERM +W0215 02:16:39.063000 125992953558848 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 368409 closing signal SIGTERM +W0215 02:16:39.063000 125992953558848 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 368410 closing signal SIGTERM +W0215 02:16:39.063000 125992953558848 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 368411 closing signal SIGTERM +W0215 02:16:39.063000 125992953558848 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 368412 closing signal SIGTERM +W0215 02:16:39.063000 125992953558848 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 368413 closing signal SIGTERM +W0215 02:16:39.063000 125992953558848 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 368414 closing signal SIGTERM +W0215 02:16:39.066000 139602503870272 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1195005 closing signal SIGTERM +W0215 02:16:39.067000 139602503870272 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1195006 closing signal SIGTERM +W0215 02:16:39.068000 139602503870272 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1195007 closing signal SIGTERM +W0215 02:16:39.069000 139602503870272 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1195008 closing signal SIGTERM +W0215 02:16:39.069000 139602503870272 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1195009 closing signal SIGTERM +W0215 02:16:39.069000 139602503870272 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1195010 closing signal SIGTERM +W0215 02:16:39.069000 139602503870272 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1195011 closing signal SIGTERM +W0215 02:16:39.368000 127727230662464 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 277820 closing signal SIGTERM +W0215 02:16:39.368000 127727230662464 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 277821 closing signal SIGTERM +W0215 02:16:39.368000 127727230662464 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 277822 closing signal SIGTERM +W0215 02:16:39.368000 124609366361920 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1195439 closing signal SIGTERM +W0215 02:16:39.368000 127727230662464 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 277823 closing signal SIGTERM +W0215 02:16:39.368000 124609366361920 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1195440 closing signal SIGTERM +W0215 02:16:39.368000 127727230662464 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 277825 closing signal SIGTERM +W0215 02:16:39.369000 127727230662464 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 277826 closing signal SIGTERM +W0215 02:16:39.369000 124609366361920 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1195441 closing signal SIGTERM +W0215 02:16:39.369000 124609366361920 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1195443 closing signal SIGTERM +W0215 02:16:39.369000 124609366361920 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1195444 closing signal SIGTERM +W0215 02:16:39.369000 124609366361920 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1195445 closing signal SIGTERM +W0215 02:16:39.369000 124609366361920 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1195446 closing signal SIGTERM +W0215 02:16:39.761000 123775382849344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 204676 closing signal SIGTERM +W0215 02:16:39.761000 123775382849344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 204677 closing signal SIGTERM +W0215 02:16:39.761000 123775382849344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 204678 closing signal SIGTERM +W0215 02:16:39.761000 123775382849344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 204679 closing signal SIGTERM +W0215 02:16:39.762000 123775382849344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 204680 closing signal SIGTERM +W0215 02:16:39.762000 123775382849344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 204682 closing signal SIGTERM +W0215 02:16:39.762000 123775382849344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 204683 closing signal SIGTERM +E0215 02:16:39.821000 134719829468992 torch/distributed/elastic/multiprocessing/api.py:833] failed (exitcode: 1) local_rank: 0 (pid: 226744) of binary: /data/home/zhaojiang/miniconda3/envs/test/bin/python3.11 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 264, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +llava/train/train_mem.py FAILED +------------------------------------------------------------ +Failures: +[1]: + time : 2025-02-15_02:16:38 + host : h100-st-p548xlarge-118.ar-ai-use2.hpcaas + rank : 66 (local_rank: 2) + exitcode : 1 (pid: 226746) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-02-15_02:16:38 + host : h100-st-p548xlarge-118.ar-ai-use2.hpcaas + rank : 64 (local_rank: 0) + exitcode : 1 (pid: 226744) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +srun: error: h100-st-p548xlarge-118: task 8: Exited with exit code 1 +srun: Terminating StepId=334457.0 +slurmstepd: error: *** STEP 334457.0 ON h100-st-p548xlarge-17 CANCELLED AT 2025-02-15T02:16:40 *** +W0215 02:16:40.158000 133614182537024 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:16:40.158000 136148163000128 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:16:40.158000 133614182537024 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1186439 closing signal SIGTERM +W0215 02:16:40.158000 136148163000128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 205124 closing signal SIGTERM +W0215 02:16:40.158000 128235731347264 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:16:40.158000 139602503870272 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:16:40.158000 139602503870272 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1195008 closing signal SIGTERM +W0215 02:16:40.158000 131818974000960 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:16:40.158000 128235731347264 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 310644 closing signal SIGTERM +W0215 02:16:40.158000 139602503870272 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1195009 closing signal SIGTERM +W0215 02:16:40.158000 131818974000960 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 821958 closing signal SIGTERM +W0215 02:16:40.158000 136148163000128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 205126 closing signal SIGTERM +W0215 02:16:40.158000 127343532558144 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:16:40.158000 127343532558144 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2949110 closing signal SIGTERM +W0215 02:16:40.158000 123775382849344 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:16:40.158000 124609366361920 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:16:40.158000 123775382849344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 204676 closing signal SIGTERM +W0215 02:16:40.158000 125992953558848 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:16:40.158000 124609366361920 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1195445 closing signal SIGTERM +W0215 02:16:40.158000 123775382849344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 204677 closing signal SIGTERM +W0215 02:16:40.158000 125992953558848 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 368407 closing signal SIGTERM +W0215 02:16:40.158000 133614182537024 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1186443 closing signal SIGTERM +W0215 02:16:40.158000 131818974000960 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 821960 closing signal SIGTERM +W0215 02:16:40.158000 125992953558848 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 368409 closing signal SIGTERM +W0215 02:16:40.158000 124877567706944 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:16:40.158000 129987096590144 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:16:40.158000 124877567706944 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 202465 closing signal SIGTERM +W0215 02:16:40.158000 128930927953728 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:16:40.158000 129987096590144 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 810035 closing signal SIGTERM +W0215 02:16:40.158000 128930927953728 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1500456 closing signal SIGTERM +W0215 02:16:40.158000 123775382849344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 204679 closing signal SIGTERM +W0215 02:16:40.158000 124877567706944 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 202467 closing signal SIGTERM +W0215 02:16:40.158000 123775382849344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 204680 closing signal SIGTERM +W0215 02:16:40.158000 131818974000960 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 821964 closing signal SIGTERM +W0215 02:16:40.159000 124877567706944 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 202468 closing signal SIGTERM +W0215 02:16:40.158000 130114260338496 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:16:40.159000 123775382849344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 204682 closing signal SIGTERM +W0215 02:16:40.159000 124877567706944 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 202469 closing signal SIGTERM +W0215 02:16:40.159000 123775382849344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 204683 closing signal SIGTERM +W0215 02:16:40.159000 130114260338496 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1194202 closing signal SIGTERM +W0215 02:16:40.158000 127727230662464 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:16:40.159000 133614182537024 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1186446 closing signal SIGTERM +W0215 02:16:40.159000 125992953558848 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 368413 closing signal SIGTERM +W0215 02:16:40.159000 127727230662464 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 277820 closing signal SIGTERM +W0215 02:16:40.159000 124877567706944 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 202471 closing signal SIGTERM +W0215 02:16:40.159000 128930927953728 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1500460 closing signal SIGTERM +W0215 02:16:40.159000 130114260338496 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1194204 closing signal SIGTERM +W0215 02:16:40.159000 124877567706944 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 202472 closing signal SIGTERM +W0215 02:16:40.159000 127727230662464 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 277822 closing signal SIGTERM +W0215 02:16:40.159000 127727230662464 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 277825 closing signal SIGTERM +W0215 02:16:40.159000 140083807930176 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:16:40.160000 140083807930176 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 205435 closing signal SIGTERM +W0215 02:16:40.160000 140083807930176 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 205436 closing signal SIGTERM +W0215 02:16:40.160000 140083807930176 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 205437 closing signal SIGTERM +W0215 02:16:40.160000 140083807930176 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 205438 closing signal SIGTERM +W0215 02:16:40.160000 140083807930176 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 205439 closing signal SIGTERM +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 835, in _invoke_run + time.sleep(monitor_interval) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 201748 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 2949024 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 1186359 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 809313 got signal: 15 +srun: error: h100-st-p548xlarge-210: task 11: Exited with exit code 1 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 368329 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 1194116 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 821249 got signal: 15 +srun: error: h100-st-p548xlarge-392: task 15: Exited with exit code 1 +srun: error: h100-st-p548xlarge-17: task 0: Exited with exit code 1 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait +srun: error: h100-st-p548xlarge-49: task 4: Exited with exit code 1 + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 310121 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 277737 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 1500379 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait +srun: error: h100-st-p548xlarge-383: task 13: Exited with exit code 1 + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 1194924 got signal: 15 +srun: error: h100-st-p548xlarge-18: task 1: Exited with exit code 1 +srun: error: h100-st-p548xlarge-52: task 5: Exited with exit code 1 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 203832 got signal: 15 +srun: error: h100-st-p548xlarge-382: task 12: Exited with exit code 1 +srun: error: h100-st-p548xlarge-117: task 7: Exited with exit code 1 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 1195360 got signal: 15 +srun: error: h100-st-p548xlarge-391: task 14: Exited with exit code 1 +srun: error: h100-st-p548xlarge-24: task 3: Exited with exit code 1 +srun: error: h100-st-p548xlarge-116: task 6: Exited with exit code 1 +srun: error: h100-st-p548xlarge-20: task 2: Exited with exit code 1 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 205357 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 205045 got signal: 15 +srun: error: h100-st-p548xlarge-208: task 9: Exited with exit code 1 +srun: error: h100-st-p548xlarge-209: task 10: Exited with exit code 1 +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/upload_pretrain.py", line 16, in + upload_folder( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn + return fn(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/huggingface_hub/hf_api.py", line 1398, in _inner + return fn(self, *args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/huggingface_hub/hf_api.py", line 4824, in upload_folder + add_operations = _prepare_upload_folder_additions( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/huggingface_hub/hf_api.py", line 9227, in _prepare_upload_folder_additions + raise ValueError(f"Provided path: '{folder_path}' is not a directory") +ValueError: Provided path: '/fsx-project/xichenpan/PISCES/latent_query_ar_20241220' is not a directory + Reformatting /fsx_0/user/zhaojiang/data/ShareGPT4V/blip_laion_cc_sbu_558k.json: 0%| | 0/558128 [00:00 + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportErrorImportError: ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLMfrom llava.train.train import train + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + + from llava.train.train import train + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .model import LlavaLlamaForCausalLM + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + Traceback (most recent call last): +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import trainfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .model import LlavaLlamaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportErrorImportErrorfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina: : ImportError +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + + from llava.train.train import train File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportErrorImportError +: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +Traceback (most recent call last): + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.train.train import train File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .model import LlavaLlamaForCausalLM + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train +from llava.train.train import train +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from llava.train.train import train + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + from llava.train.train import train from llava.train.train import train +from llava.train.train import train + + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + from .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: ImportErrorImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): ImportError: ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): +: +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportError File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:47: FutureWarning: `torch.cuda.amp.custom_fwd(args...)` is deprecated. Please use `torch.amp.custom_fwd(args..., device_type='cuda')` instead. + @autocast_custom_fwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/deepspeed/runtime/zero/linear.py:66: FutureWarning: `torch.cuda.amp.custom_bwd(args...)` is deprecated. Please use `torch.amp.custom_bwd(args..., device_type='cuda')` instead. + @autocast_custom_bwd +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .model import LlavaLlamaForCausalLMfrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import trainfrom llava.train.train import train + + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.train.train import train File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM +ImportError File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.train.train import train + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .model import LlavaLlamaForCausalLMfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError : from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminacannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.train.train import trainImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +W0215 02:22:40.375000 126765361080128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 209252 closing signal SIGTERM +W0215 02:22:40.375000 126765361080128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 209253 closing signal SIGTERM +W0215 02:22:40.375000 126765361080128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 209254 closing signal SIGTERM +W0215 02:22:40.375000 126765361080128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 209256 closing signal SIGTERM +W0215 02:22:40.376000 126765361080128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 209257 closing signal SIGTERM +W0215 02:22:40.376000 126765361080128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 209258 closing signal SIGTERM +W0215 02:22:40.376000 126765361080128 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 209259 closing signal SIGTERM +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import trainfrom llava.train.train import train + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.train.train import train + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + ImportError: from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminacannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom .model import LlavaLlamaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportError +ImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + from llava.train.train import train + from llava.train.train import trainfrom llava.train.train import train + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from llava.train.train import train + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .model import LlavaLlamaForCausalLM +from .model import LlavaLlamaForCausalLMfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + + +from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: ImportErrorImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py): : +cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)ImportError + +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_ditfrom .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminafrom diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + +ImportErrorImportError: : cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py)cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + from llava.train.train import train + from llava.train.train import trainfrom llava.train.train import train + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from llava.train.train import trainfrom llava.train.train import train + +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLMfrom .model import LlavaLlamaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttnfrom llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +Traceback (most recent call last): +Traceback (most recent call last): +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + Traceback (most recent call last): + from llava.train.train import trainfrom llava.train.train import trainfrom llava.train.train import train + + +from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +Traceback (most recent call last): + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/train/train_mem.py", line 1, in + from llava.train.train import train + from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in + from .model import LlavaLlamaForCausalLM from .model import LlavaLlamaForCausalLM + from .model import LlavaLlamaForCausalLM + +from .model import LlavaLlamaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + from .model import LlavaLlamaForCausalLM + from llava.train.train import train + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/__init__.py", line 1, in +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfigfrom .model import LlavaLlamaForCausalLMfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig +from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + from .model import LlavaLlamaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + + +from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/__init__.py", line 2, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLMfrom .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig + + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/language_model/llava_llama.py", line 29, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/llava_arch.py", line 22, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn +from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from .multimodal_encoder.builder import build_vision_tower, build_gen_vision_tower, build_dit + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/multimodal_encoder/builder.py", line 12, in + from llava.model.nextdit_crossattn import NextDiTCrossAttnConfig, NextDiTCrossAttn + File "/opt/hpcaas/.mounts/fs-036153e63d56f4dc2/home/zhaojiang/interleaved-llava/llava/model/nextdit_crossattn.py", line 4, in + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +from diffusers.models.embeddings import get_2d_rotary_pos_embed_luminaImportError +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +ImportError: ImportErrorcannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) + from diffusers.models.embeddings import get_2d_rotary_pos_embed_lumina +ImportError: cannot import name 'get_2d_rotary_pos_embed_lumina' from 'diffusers.models.embeddings' (/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/diffusers/models/embeddings.py) +W0215 02:22:41.985000 125660924434240 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 211959 closing signal SIGTERM +W0215 02:22:41.986000 125660924434240 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 211960 closing signal SIGTERM +W0215 02:22:41.986000 125660924434240 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 211961 closing signal SIGTERM +W0215 02:22:41.987000 125660924434240 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 211963 closing signal SIGTERM +W0215 02:22:41.987000 125660924434240 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 211964 closing signal SIGTERM +W0215 02:22:41.987000 125660924434240 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 211966 closing signal SIGTERM +W0215 02:22:41.986000 132066791716672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1509268 closing signal SIGTERM +W0215 02:22:41.988000 132066791716672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1509269 closing signal SIGTERM +W0215 02:22:41.989000 132066791716672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1509270 closing signal SIGTERM +W0215 02:22:41.989000 132066791716672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1509272 closing signal SIGTERM +W0215 02:22:41.990000 132066791716672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1509273 closing signal SIGTERM +W0215 02:22:41.990000 132066791716672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1509274 closing signal SIGTERM +W0215 02:22:41.991000 132066791716672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1509275 closing signal SIGTERM +W0215 02:22:42.079000 131900223469376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 827855 closing signal SIGTERM +W0215 02:22:42.080000 131900223469376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 827856 closing signal SIGTERM +W0215 02:22:42.080000 131900223469376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 827857 closing signal SIGTERM +W0215 02:22:42.081000 131900223469376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 827858 closing signal SIGTERM +W0215 02:22:42.081000 131900223469376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 827859 closing signal SIGTERM +W0215 02:22:42.082000 131900223469376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 827860 closing signal SIGTERM +W0215 02:22:42.082000 131900223469376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 827862 closing signal SIGTERM +W0215 02:22:42.082000 134109160027968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 211459 closing signal SIGTERM +W0215 02:22:42.083000 134109160027968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 211460 closing signal SIGTERM +W0215 02:22:42.084000 134109160027968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 211461 closing signal SIGTERM +W0215 02:22:42.086000 134109160027968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 211462 closing signal SIGTERM +W0215 02:22:42.086000 134109160027968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 211463 closing signal SIGTERM +W0215 02:22:42.087000 134109160027968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 211465 closing signal SIGTERM +W0215 02:22:42.087000 134109160027968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 211466 closing signal SIGTERM +W0215 02:22:42.286000 124346442655552 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 317941 closing signal SIGTERM +W0215 02:22:42.287000 124346442655552 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 317942 closing signal SIGTERM +W0215 02:22:42.288000 124346442655552 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 317943 closing signal SIGTERM +W0215 02:22:42.289000 124346442655552 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 317944 closing signal SIGTERM +W0215 02:22:42.289000 124346442655552 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 317946 closing signal SIGTERM +W0215 02:22:42.290000 124346442655552 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 317947 closing signal SIGTERM +W0215 02:22:42.291000 124346442655552 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 317948 closing signal SIGTERM +W0215 02:22:42.377000 136818624038720 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2956429 closing signal SIGTERM +W0215 02:22:42.377000 136818624038720 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2956430 closing signal SIGTERM +W0215 02:22:42.377000 136818624038720 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2956431 closing signal SIGTERM +W0215 02:22:42.377000 136818624038720 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2956432 closing signal SIGTERM +W0215 02:22:42.378000 136818624038720 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2956433 closing signal SIGTERM +W0215 02:22:42.387000 124727191713600 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 375020 closing signal SIGTERM +W0215 02:22:42.388000 124727191713600 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 375021 closing signal SIGTERM +W0215 02:22:42.389000 124727191713600 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 375022 closing signal SIGTERM +W0215 02:22:42.389000 124727191713600 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 375023 closing signal SIGTERM +W0215 02:22:42.389000 124727191713600 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 375024 closing signal SIGTERM +W0215 02:22:42.390000 124727191713600 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 375025 closing signal SIGTERM +W0215 02:22:42.390000 124727191713600 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 375026 closing signal SIGTERM +W0215 02:22:42.379000 136818624038720 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2956434 closing signal SIGTERM +W0215 02:22:42.390000 136818624038720 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2956435 closing signal SIGTERM +W0215 02:22:42.485000 128858743379776 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 815956 closing signal SIGTERM +W0215 02:22:42.486000 128858743379776 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 815957 closing signal SIGTERM +W0215 02:22:42.486000 128858743379776 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 815958 closing signal SIGTERM +W0215 02:22:42.487000 128858743379776 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 815959 closing signal SIGTERM +W0215 02:22:42.487000 128858743379776 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 815960 closing signal SIGTERM +W0215 02:22:42.487000 128858743379776 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 815961 closing signal SIGTERM +W0215 02:22:42.488000 128858743379776 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 815963 closing signal SIGTERM +W0215 02:22:42.590000 136370367297344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 233558 closing signal SIGTERM +W0215 02:22:42.591000 136370367297344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 233559 closing signal SIGTERM +W0215 02:22:42.591000 136370367297344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 233560 closing signal SIGTERM +W0215 02:22:42.591000 136370367297344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 233561 closing signal SIGTERM +W0215 02:22:42.591000 136370367297344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 233563 closing signal SIGTERM +W0215 02:22:42.592000 136370367297344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 233564 closing signal SIGTERM +W0215 02:22:42.592000 136370367297344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 233565 closing signal SIGTERM +W0215 02:22:42.687000 123443012360000 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1202746 closing signal SIGTERM +W0215 02:22:42.689000 123443012360000 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1202747 closing signal SIGTERM +W0215 02:22:42.690000 123443012360000 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1202748 closing signal SIGTERM +W0215 02:22:42.690000 123443012360000 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1202749 closing signal SIGTERM +W0215 02:22:42.691000 123443012360000 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1202750 closing signal SIGTERM +W0215 02:22:42.692000 123443012360000 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1202751 closing signal SIGTERM +W0215 02:22:42.692000 123443012360000 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1202752 closing signal SIGTERM +W0215 02:22:42.780000 126960776755008 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 212251 closing signal SIGTERM +W0215 02:22:42.782000 126960776755008 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 212252 closing signal SIGTERM +W0215 02:22:42.782000 126960776755008 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 212253 closing signal SIGTERM +W0215 02:22:42.783000 126960776755008 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 212255 closing signal SIGTERM +W0215 02:22:42.783000 126960776755008 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 212256 closing signal SIGTERM +W0215 02:22:42.784000 126960776755008 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 212257 closing signal SIGTERM +W0215 02:22:42.784000 126960776755008 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 212258 closing signal SIGTERM +E0215 02:22:42.845000 126765361080128 torch/distributed/elastic/multiprocessing/api.py:833] failed (exitcode: 1) local_rank: 3 (pid: 209255) of binary: /data/home/zhaojiang/miniconda3/envs/test/bin/python3.11 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 264, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +llava/train/train_mem.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2025-02-15_02:22:40 + host : h100-st-p548xlarge-210.ar-ai-use2.hpcaas + rank : 91 (local_rank: 3) + exitcode : 1 (pid: 209255) + error_file: + traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html +============================================================ +W0215 02:22:42.879000 129789458925376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1200935 closing signal SIGTERM +W0215 02:22:42.880000 129789458925376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1200936 closing signal SIGTERM +W0215 02:22:42.881000 129789458925376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1200937 closing signal SIGTERM +W0215 02:22:42.881000 129789458925376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1200938 closing signal SIGTERM +W0215 02:22:42.881000 129789458925376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1200939 closing signal SIGTERM +W0215 02:22:42.882000 129789458925376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1200941 closing signal SIGTERM +W0215 02:22:42.882000 129789458925376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1200942 closing signal SIGTERM +W0215 02:22:43.093000 126594262841152 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1201566 closing signal SIGTERM +W0215 02:22:43.095000 126594262841152 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1201568 closing signal SIGTERM +W0215 02:22:43.096000 126594262841152 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1201569 closing signal SIGTERM +W0215 02:22:43.096000 126594262841152 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1201571 closing signal SIGTERM +W0215 02:22:43.096000 126594262841152 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1201572 closing signal SIGTERM +W0215 02:22:43.097000 126594262841152 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1201573 closing signal SIGTERM +W0215 02:22:43.096000 136955260069696 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 286008 closing signal SIGTERM +W0215 02:22:43.097000 136955260069696 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 286009 closing signal SIGTERM +W0215 02:22:43.098000 136955260069696 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 286010 closing signal SIGTERM +W0215 02:22:43.098000 136955260069696 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 286011 closing signal SIGTERM +W0215 02:22:43.099000 136955260069696 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 286012 closing signal SIGTERM +W0215 02:22:43.099000 136955260069696 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 286013 closing signal SIGTERM +W0215 02:22:43.099000 136955260069696 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 286014 closing signal SIGTERM +srun: error: h100-st-p548xlarge-210: task 11: Exited with exit code 1 +srun: Terminating StepId=334457.1 +W0215 02:22:43.182000 124631130990400 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1193794 closing signal SIGTERM +W0215 02:22:43.184000 131900223469376 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:22:43.184000 124727191713600 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:22:43.184000 131900223469376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 827855 closing signal SIGTERM +W0215 02:22:43.184000 125660924434240 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:22:43.184000 136818624038720 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:22:43.184000 132066791716672 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:22:43.184000 124727191713600 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 375021 closing signal SIGTERM +W0215 02:22:43.184000 124631130990400 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1193795 closing signal SIGTERM +W0215 02:22:43.184000 131900223469376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 827856 closing signal SIGTERM +W0215 02:22:43.185000 125660924434240 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 211959 closing signal SIGTERM +W0215 02:22:43.184000 136818624038720 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2956431 closing signal SIGTERM +W0215 02:22:43.185000 132066791716672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1509273 closing signal SIGTERM +W0215 02:22:43.184000 124727191713600 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 375022 closing signal SIGTERM +W0215 02:22:43.185000 123443012360000 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +slurmstepd: error: *** STEP 334457.1 ON h100-st-p548xlarge-17 CANCELLED AT 2025-02-15T02:22:43 *** +W0215 02:22:43.185000 131900223469376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 827862 closing signal SIGTERM +W0215 02:22:43.185000 136818624038720 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2956432 closing signal SIGTERM +W0215 02:22:43.185000 132066791716672 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1509274 closing signal SIGTERM +W0215 02:22:43.184000 124727191713600 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 375023 closing signal SIGTERM +W0215 02:22:43.185000 123443012360000 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1202746 closing signal SIGTERM +W0215 02:22:43.185000 136818624038720 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 2956433 closing signal SIGTERM +W0215 02:22:43.185000 124727191713600 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 375025 closing signal SIGTERM +W0215 02:22:43.185000 123443012360000 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1202747 closing signal SIGTERM +W0215 02:22:43.185000 124631130990400 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1193796 closing signal SIGTERM +W0215 02:22:43.186000 126960776755008 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:22:43.185000 124346442655552 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:22:43.185000 124727191713600 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 375026 closing signal SIGTERM +W0215 02:22:43.185000 123443012360000 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1202748 closing signal SIGTERM +W0215 02:22:43.186000 126960776755008 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 212251 closing signal SIGTERM +W0215 02:22:43.186000 124346442655552 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 317941 closing signal SIGTERM +W0215 02:22:43.185000 123443012360000 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1202749 closing signal SIGTERM +W0215 02:22:43.186000 128858743379776 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:22:43.186000 134109160027968 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:22:43.186000 126960776755008 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 212252 closing signal SIGTERM +W0215 02:22:43.186000 124346442655552 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 317942 closing signal SIGTERM +W0215 02:22:43.186000 123443012360000 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1202750 closing signal SIGTERM +W0215 02:22:43.186000 128858743379776 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 815957 closing signal SIGTERM +W0215 02:22:43.186000 134109160027968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 211459 closing signal SIGTERM +W0215 02:22:43.186000 126960776755008 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 212253 closing signal SIGTERM +W0215 02:22:43.186000 124346442655552 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 317943 closing signal SIGTERM +W0215 02:22:43.186000 123443012360000 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1202751 closing signal SIGTERM +W0215 02:22:43.187000 128858743379776 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 815958 closing signal SIGTERM +W0215 02:22:43.186000 134109160027968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 211460 closing signal SIGTERM +W0215 02:22:43.186000 126960776755008 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 212255 closing signal SIGTERM +W0215 02:22:43.186000 136370367297344 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:22:43.186000 124346442655552 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 317944 closing signal SIGTERM +W0215 02:22:43.187000 128858743379776 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 815959 closing signal SIGTERM +W0215 02:22:43.187000 134109160027968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 211463 closing signal SIGTERM +W0215 02:22:43.186000 126960776755008 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 212256 closing signal SIGTERM +W0215 02:22:43.187000 136370367297344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 233558 closing signal SIGTERM +W0215 02:22:43.186000 124346442655552 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 317946 closing signal SIGTERM +W0215 02:22:43.186000 126594262841152 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:22:43.187000 128858743379776 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 815960 closing signal SIGTERM +W0215 02:22:43.187000 134109160027968 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 211466 closing signal SIGTERM +W0215 02:22:43.186000 126960776755008 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 212257 closing signal SIGTERM +W0215 02:22:43.187000 124631130990400 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:22:43.186000 126960776755008 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 212258 closing signal SIGTERM +W0215 02:22:43.187000 136370367297344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 233559 closing signal SIGTERM +W0215 02:22:43.188000 136370367297344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 233560 closing signal SIGTERM +W0215 02:22:43.188000 136370367297344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 233561 closing signal SIGTERM +W0215 02:22:43.188000 126594262841152 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1201568 closing signal SIGTERM +W0215 02:22:43.188000 124631130990400 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1193794 closing signal SIGTERM +W0215 02:22:43.188000 136370367297344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 233563 closing signal SIGTERM +W0215 02:22:43.188000 136370367297344 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 233564 closing signal SIGTERM +W0215 02:22:43.188000 126594262841152 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1201569 closing signal SIGTERM +W0215 02:22:43.189000 126594262841152 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1201571 closing signal SIGTERM +W0215 02:22:43.188000 124631130990400 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1193795 closing signal SIGTERM +W0215 02:22:43.189000 126594262841152 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1201572 closing signal SIGTERM +W0215 02:22:43.188000 129789458925376 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:22:43.189000 126594262841152 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1201573 closing signal SIGTERM +W0215 02:22:43.189000 129789458925376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1200935 closing signal SIGTERM +W0215 02:22:43.189000 124631130990400 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1193796 closing signal SIGTERM +W0215 02:22:43.190000 129789458925376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1200936 closing signal SIGTERM +W0215 02:22:43.189000 136955260069696 torch/distributed/elastic/agent/server/api.py:688] Received 15 death signal, shutting down workers +W0215 02:22:43.190000 124631130990400 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1193797 closing signal SIGTERM +W0215 02:22:43.190000 129789458925376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1200937 closing signal SIGTERM +W0215 02:22:43.190000 136955260069696 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 286008 closing signal SIGTERM +W0215 02:22:43.191000 129789458925376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1200938 closing signal SIGTERM +W0215 02:22:43.190000 124631130990400 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1193798 closing signal SIGTERM +W0215 02:22:43.191000 129789458925376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1200939 closing signal SIGTERM +W0215 02:22:43.191000 124631130990400 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1193800 closing signal SIGTERM +W0215 02:22:43.191000 136955260069696 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 286009 closing signal SIGTERM +W0215 02:22:43.191000 136955260069696 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 286010 closing signal SIGTERM +W0215 02:22:43.191000 129789458925376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1200941 closing signal SIGTERM +W0215 02:22:43.191000 124631130990400 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1193801 closing signal SIGTERM +W0215 02:22:43.192000 129789458925376 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 1200942 closing signal SIGTERM +W0215 02:22:43.192000 136955260069696 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 286011 closing signal SIGTERM +W0215 02:22:43.192000 136955260069696 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 286012 closing signal SIGTERM +W0215 02:22:43.193000 136955260069696 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 286013 closing signal SIGTERM +W0215 02:22:43.193000 136955260069696 torch/distributed/elastic/multiprocessing/api.py:858] Sending process 286014 closing signal SIGTERM +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 1508294 got signal: 15 +srun: error: h100-st-p548xlarge-391: task 14: Exited with exit code 1 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 827780 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 2956362 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 211893 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 211393 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 374305 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 815886 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 1202678 got signal: 15 +srun: error: h100-st-p548xlarge-52: task 5: Exited with exit code 1 +srun: error: h100-st-p548xlarge-392: task 15: Exited with exit code 1 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 317873 got signal: 15 +srun: error: h100-st-p548xlarge-208: task 9: Exited with exit code 1 +srun: error: h100-st-p548xlarge-383: task 13: Exited with exit code 1 +srun: error: h100-st-p548xlarge-116: task 6: Exited with exit code 1 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 212186 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 233492 got signal: 15 +srun: error: h100-st-p548xlarge-49: task 4: Exited with exit code 1 +srun: error: h100-st-p548xlarge-20: task 2: Exited with exit code 1 +srun: error: h100-st-p548xlarge-382: task 12: Exited with exit code 1 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 1201498 got signal: 15 +srun: error: h100-st-p548xlarge-209: task 10: Exited with exit code 1 +srun: error: h100-st-p548xlarge-118: task 8: Exited with exit code 1 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 858, in _close + logger.warning( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/logging/__init__.py", line 1501, in warning + self._log(WARNING, msg, args, **kwargs) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/logging/__init__.py", line 1632, in _log + record = self.makeRecord(self.name, level, fn, lno, msg, args, + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/logging/__init__.py", line 1601, in makeRecord + rv = _logRecordFactory(name, level, fn, lno, msg, args, exc_info, func, + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/logging/__init__.py", line 322, in __init__ + self.levelname = getLevelName(level) + ^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/logging/__init__.py", line 142, in getLevelName + result = _levelToName.get(level) + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 1193728 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 285941 got signal: 15 +Traceback (most recent call last): + File "/data/home/zhaojiang/miniconda3/envs/test/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 348, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 901, in main + run(args) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in run + elastic_launch( + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 133, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 255, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 680, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 836, in _invoke_run + run_result = self._monitor_workers(self._worker_group) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 124, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/local_elastic_agent.py", line 367, in _monitor_workers + result = self._pcontext.wait(0) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 503, in wait + return self._poll() + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 825, in _poll + self.close() # terminate all running procs + ^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 544, in close + self._close(death_sig=death_sig, timeout=timeout) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 868, in _close + handler.proc.wait(time_to_wait) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 1264, in wait + return self._wait(timeout=timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/subprocess.py", line 2047, in _wait + time.sleep(delay) + File "/data/home/zhaojiang/miniconda3/envs/test/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py", line 79, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 1200867 got signal: 15 +srun: error: h100-st-p548xlarge-18: task 1: Exited with exit code 1 +srun: error: h100-st-p548xlarge-117: task 7: Exited with exit code 1 +srun: error: h100-st-p548xlarge-24: task 3: Exited with exit code 1 +srun: error: h100-st-p548xlarge-17: task 0: Exited with exit code 1